{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## IMPORTANT!!\n",
    "\n",
    "### From the root NetworkML directory, make sure you first run:\n",
    "```\n",
    "pip3 install .\n",
    "pip3 install nest_asyncio\n",
    "```\n",
    "\n",
    "NOTE: nest_asyncio is a needed hack to run this in jupyterlab/notebook. Additionally, you will also need to download Wireshark/tshark in order to use this notebook and then ensure that tshark is in the file path."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# some initial setup\n",
    "import sys\n",
    "\n",
    "# let's set a path to a pcap (we'll use one included in the tests)\n",
    "path = '../tests/test_data/trace_ab12_2001-01-01_02_03-client-ip-1-2-3-4.pcap'\n",
    "\n",
    "# let's change the output so it's easy to find\n",
    "output = 'trace_ab12_2001-01-01_02_03-client-ip-1-2-3-4.pcap.csv.gz'\n",
    "\n",
    "# set arguments for arg parse\n",
    "sys.argv = ['pcap_to_csv.py', f'-o{output}', path]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:networkml.parsers.pcap_to_csv:Processing ../tests/test_data/trace_ab12_2001-01-01_02_03-client-ip-1-2-3-4.pcap\n",
      "INFO:networkml.parsers.pcap_to_csv:Finished ../tests/test_data/trace_ab12_2001-01-01_02_03-client-ip-1-2-3-4.pcap. 1/1 PCAPs done.\n",
      "INFO:networkml.parsers.pcap_to_csv:GZipped CSV file(s) written out to: ['trace_ab12_2001-01-01_02_03-client-ip-1-2-3-4.pcap.csv.gz']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "''"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# hack for jupyterlab since pyshark tries to take the main run loop\n",
    "import nest_asyncio\n",
    "nest_asyncio.apply()\n",
    "\n",
    "# import the class for converting PCAPs to CSVs\n",
    "from networkml.parsers.pcap_to_csv import PCAPToCSV\n",
    "instance = PCAPToCSV()\n",
    "\n",
    "# this will parse the args we specified above in sys.argv and run using them\n",
    "instance.main()\n",
    "\n",
    "# the will take a minute or so to run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OrderedDict([('ip.id', '0x000076e0'), ('dns.qry.name.len', ''), ('smb.access.delete_child', ''), ('smb.access.read_control', ''), ('dns.resp.len', ''), ('smb.nt.create_options.non_directory', ''), ('smb.pc', ''), ('bootp.option.type', ''), ('ssl.handshake.extensions_status_request_exts_len', ''), ('tcp.window_size_scalefactor', '-1'), ('tcp.connection.fin', ''), ('nbdgm.src.ip', ''), ('ip.flags', '0x00004000'), ('smb.nt.create_options.reserve_opfilter', ''), ('eth.lg', '0'), ('http.request.uri.query.parameter', 'q=cr'), ('http.chunk_size', ''), ('rss.channel.image.title', ''), ('smb.pid.high', ''), ('nbns.count.answers', ''), ('ssl.handshake.extensions_status_request_type', ''), ('frame.time_relative', '0.000000000'), ('tcp.checksum', '0x00005df8'), ('ssl.handshake.version', ''), ('snmp.error_index', ''), ('ssl.handshake.extensions_ec_point_formats_length', ''), ('png.phys.vert', ''), ('browser.server_type.dialin', ''), ('dns.retransmit_request', ''), ('frame.cap_len', '997'), ('smb.timeout', ''), ('arp.src.proto_ipv4', ''), ('rss.version', ''), ('smb.nt.create_options.sync_io_nonalert', ''), ('smb.nt.create_options.no_compression', ''), ('arp.hw.type', ''), ('nat-pmp.opcode', ''), ('ssl.handshake.extensions_reneg_info_len', ''), ('smb.security.flags.context_tracking', ''), ('ssl.handshake.random', ''), ('ip.src', '192.168.3.131'), ('ssl.handshake.random_bytes', ''), ('http.set_cookie', ''), ('xml.tag', ''), ('browser.proto_minor', ''), ('rss.channel.link', ''), ('ip.flags.mf', '0'), ('image-jfif.sos.ac_entropy_selector', ''), ('dns.resp.type', ''), ('ip.dsfield.dscp', '0'), ('ssl.handshake.cipher_suites_length', ''), ('nbns.count.queries', ''), ('ssl.handshake.type', ''), ('tcp.option_len', ''), ('tcp.analysis.rto_frame', ''), ('dns.count.auth_rr', ''), ('http.file_data', ''), ('http.transfer_encoding', ''), ('smb.access.system_security', ''), ('tcp.time_relative', '0.000000000'), ('tcp.port', '80'), ('http.request_in', ''), ('browser.command', ''), ('tcp.connection.rst', ''), ('arp.dst.proto_ipv4', ''), ('udp.checksum.status', ''), ('ssl.handshake', ''), ('icmp.seq_le', ''), ('ssl.handshake.extensions_server_name', ''), ('smb.andxoffset', ''), ('rss.channel.item.enclosure.length', ''), ('xml.xmlpi.xml.version', ''), ('ip.checksum.status', '2'), ('rss.channel.language', ''), ('tcp.srcport', '57011'), ('image-gif.image.height', ''), ('smb.flags.lock', ''), ('smb.nt.create_options.open_by_fileid', ''), ('bootp.flags', ''), ('http.response.code.desc', ''), ('smb.echo.data', ''), ('ip.dst_host', '72.14.213.138'), ('tcp.window_size_value', '16288'), ('image-gif.global.color_map.present', ''), ('smb.cmd', ''), ('smb.nt.create_options.write_through', ''), ('_ws.malformed', ''), ('rss.channel.item.description', ''), ('smb.flags2.nt_error', ''), ('nbns.count.add_rr', ''), ('tcp.stream', '0'), ('image-jfif.sos.component_selector', ''), ('browser.server_type', ''), ('ssl.handshake.extension.len', ''), ('_ws.expert.message', 'GET /complete/search?client=chrome&hl=en-US&q=cr HTTP/1.1\\\\r\\\\n'), ('data.data', ''), ('smb.mid', ''), ('smb.flags2.compressed', ''), ('smb.trans_name', ''), ('nbdgm.source_name', ''), ('tcp.flags.ns', '0'), ('arp.proto.type', ''), ('dns.flags.authoritative', ''), ('eth.src_resolved', '40:61:86:9a:f1:f5'), ('http.response_number', ''), ('tcp.analysis.ack_rtt', ''), ('ip.ttl.too_small', ''), ('tcp.dstport', '80'), ('mailslot.opcode', ''), ('data-text-lines', ''), ('bootp.type', ''), ('dns.count.add_rr', ''), ('tcp.flags.cwr', '0'), ('tcp.analysis.bytes_in_flight', '943'), ('ip.dsfield.ecn', '0'), ('smb.share.access.delete', ''), ('nbns.name', ''), ('browser.server_type.apple', ''), ('smb.file_attribute.reparse', ''), ('ssl.handshake.extension.type', ''), ('_ws.malformed.expert', ''), ('bootp.option.padding', ''), ('smb.file', ''), ('ssl.handshake.session_id_length', ''), ('ip.proto', '6'), ('tcp.analysis.duplicate_ack_frame', ''), ('http.response_for.uri', ''), ('rss.channel', ''), ('smb.file_attribute.encrypted', ''), ('ssl.handshake.cipher_spec_len', ''), ('browser.server_type.domain_controller', ''), ('http.request_number', '1'), ('browser.server', ''), ('png.chunk.data', ''), ('dns.flags.opcode', ''), ('icmp.unused', ''), ('image-gif.image.top', ''), ('arp.proto.size', ''), ('image-gif.image.width', ''), ('bootp.ip.your', ''), ('ssl.handshake.cipherspec', ''), ('image-gif.image_background_index', ''), ('browser.server_type.wfw', ''), ('frame.time_delta_displayed', '0.000000000'), ('ssl.handshake.extensions_supported_group', ''), ('smb.flags2.reparse_path', ''), ('bootp.hops', ''), ('smb.msc', ''), ('smb.flags2.long_names_used', ''), ('image-jfif.remain_seg_data', ''), ('smb.access.maximum_allowed', ''), ('dns.flags', ''), ('icmp.ident', ''), ('tcp.options.eol', ''), ('nbss.flags.e', ''), ('bootp.flags.bc', ''), ('image-jfif.sof.precision', ''), ('png.ihdr.compression_method', ''), ('snmp.error_status', ''), ('rss.channel.description', ''), ('ip.hdr_len', '20'), ('ip.len', '983'), ('snmp.variable_bindings', ''), ('browser.server_type.osf', ''), ('image-gif.screen.height', ''), ('bootp.hw.type', ''), ('smb.server_component', ''), ('frame.encap_type', '1'), ('tcp.reassembled.length', ''), ('frame.number', '1'), ('bootp.hw.addr_padding', ''), ('smb.create_options', ''), ('nat-pmp.reserved', ''), ('nbns.flags.broadcast', ''), ('tcp.seq', '1'), ('nbdgm.flags', ''), ('image-gif.global.bpp', ''), ('smb.access.synchronize', ''), ('tcp.analysis.window_update', ''), ('arp.hw.size', ''), ('smb.transaction.flags.owt', ''), ('tcp.flags', '0x00000018'), ('browser.server_type.w95', ''), ('nat-pmp.version', ''), ('tcp.flags.ecn', '0'), ('json.key', ''), ('png.ihdr.bitdepth', ''), ('http.request', '1'), ('http.content_encoding', ''), ('mailslot.name', ''), ('smb.sc', ''), ('ssl.ignored_unknown_record', ''), ('tcp.segment', ''), ('browser.server_type.browser.master', ''), ('frame.ignored', '0'), ('http.cache_control', ''), ('dns.resp.class', ''), ('rss.channel.lastbuilddate', ''), ('image-jfif.sof.nf', ''), ('image-gif.version', ''), ('browser.sig', ''), ('bootp.id', ''), ('smb.flags2.long_names_allowed', ''), ('frame.offset_shift', '0.000000000'), ('image-gif.local.color_bpp', ''), ('tcp.flags.syn', '0'), ('udp.port', ''), ('ssl.handshake.extensions_server_name_type', ''), ('icmp.resptime', ''), ('smb.access.generic_all', ''), ('smb.nt.create_options.eight_dot_three_only', ''), ('smb.nt.create.oplock', ''), ('dns.flags.recdesired', ''), ('http.response.version', ''), ('mailslot.size', ''), ('tcp.ack.nonzero', ''), ('rtcp.pt', ''), ('dns.qry.name', ''), ('nat-pmp.internal_port', ''), ('nbdgm.next', ''), ('browser.server_type.workstation', ''), ('png.gama.gamma', ''), ('image-jfif.sos.dc_entropy_selector', ''), ('mailslot.class', ''), ('nbdgm.node_type', ''), ('browser.proto_major', ''), ('image-gif.data_block', ''), ('tcp.nxtseq', '944'), ('http.user_agent', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.237 Safari/534.10'), ('smb.access.write_attributes', ''), ('dns.retransmit_request_in', ''), ('rss.channel.item.enclosure.url', ''), ('image-jfif.marker', ''), ('nbss.length', ''), ('smb.impersonation.level', ''), ('bootp.ip.relay', ''), ('ip.addr', '72.14.213.138'), ('smb.access.generic_write', ''), ('smb.flags2.roe', ''), ('ssl.handshake.extensions_server_name_len', ''), ('smb.access.read_attributes', ''), ('smb.nt.create_options.sequential_only', ''), ('browser.server_type.local', ''), ('tcp.segment.count', ''), ('smb.flags2.sec_sig', ''), ('browser.os_minor', ''), ('smb.nt.create_options.sync_io_alert', ''), ('tcp.connection.sack', ''), ('eth.ig', '0'), ('rss.channel.title', ''), ('dns.flags.rcode', ''), ('browser.server_type.browser.domain_master', ''), ('smb.flags2.sec_sig_required', ''), ('snmp.data', ''), ('image-jfif.sof.lines', ''), ('http.content_length', ''), ('image-jfif.sos.ns', ''), ('nbns.flags', ''), ('snmp.community', ''), ('browser.server_type.dfs', ''), ('dns.flags.conflict', ''), ('image-jfif.sos.ah', ''), ('ssl.handshake.extensions_npn_str_len', ''), ('smb.file_name_len', ''), ('smb.tid', ''), ('nbns.flags.response', ''), ('dns.qry.class', ''), ('browser.server_type.backup_controller', ''), ('ssl.handshake.session_id', ''), ('png.ihdr.height', ''), ('image-gif.screen.width', ''), ('snmp.value.octets', ''), ('rss.channel.image.link', ''), ('rss.channel.ttl', ''), ('tcp.checksum.status', '2'), ('browser.comment', ''), ('smb.echo.seq_num', ''), ('ssl.handshake.comp_method', ''), ('image-gif.end', ''), ('image-jfif.sof.samples_per_line', ''), ('dns.flags.z', ''), ('eth.src', '40:61:86:9a:f1:f5'), ('browser.update_count', ''), ('http.response', ''), ('smb.access.write', ''), ('xml.attribute', ''), ('rtcp.mediassrc', ''), ('tcp.flags.reset', '0'), ('dns.resp.name', ''), ('smb.flags', ''), ('frame.time', 'Jan 26, 2011 07:52:22.484409000 NZDT'), ('urlencoded-form.value', ''), ('browser.server_type.domainenum', ''), ('snmp.value.int', ''), ('frame.time_delta', '0.000000000'), ('smb.nt.create_options.open_reparse_point', ''), ('image-jfif.sof.tq_i', ''), ('smb.file_attribute.hidden', ''), ('ssl', ''), ('http.content_type', ''), ('ssl.handshake.extensions_length', ''), ('dns.a', ''), ('dns.retransmission', ''), ('_ws.expert.group', '33554432'), ('smb.flags.canon', ''), ('browser.windows_version', ''), ('smb.transaction.flags', ''), ('tcp.connection.syn', ''), ('http.accept_language', 'en-US,en;q=0.8'), ('tcp.analysis.retransmission', ''), ('smb.mpc', ''), ('tcp.urgent_pointer', '0'), ('http.request.uri.query', 'client=chrome&hl=en-US&q=cr'), ('smb.nt.create_options.complete_if_oplocked', ''), ('http.response.code', ''), ('browser.server_type.vms', ''), ('smb.nt.create_options.directory', ''), ('smb.flags2.string', ''), ('png.signature', ''), ('tcp.analysis.duplicate_ack_num', ''), ('smb.access.append', ''), ('ip.dsfield', '0x00000000'), ('http.content_length_header', ''), ('ssl.handshake.extensions_supported_groups_length', ''), ('rtcp.senderssrc', ''), ('rss.channel.item.enclosure', ''), ('smb.file_attribute.read_only', ''), ('smb.access.generic_read', ''), ('xml.cdata', ''), ('ip.frag_offset', '0'), ('smb.reserved', ''), ('nbdgm.first', ''), ('smb.access.generic_execute', ''), ('smb.nt.create.batch_oplock', ''), ('smb.file_attribute.volume', ''), ('http.location', ''), ('smb.nt.create.dir', ''), ('image-gif.local.color_map.present', ''), ('smb.nt.create_options.delete_on_close', ''), ('smb.flags.response', ''), ('ip.flags.df', '1'), ('smb.uid', ''), ('ip.version', '4'), ('dns.count.queries', ''), ('smb.access.execute', ''), ('smb.nt.create.ext', ''), ('eth.padding', ''), ('tcp.ack', '1'), ('ip.src_host', '192.168.3.131'), ('nbss.flags', ''), ('tcp.options.sack_re', ''), ('nat-pmp.pml', ''), ('tcp.flags.res', '0'), ('png.phys.horiz', ''), ('snmp.version', ''), ('browser.server_type.time', ''), ('smb.access.read', ''), ('http.cookie', 'PREF=ID=c2e350012258df1c:U=386a6ebef0db287c:FF=0:TM=1294164294:LM=1294164294:S=bcuwM6Vh5ecKxqmk; SID=DQAAAN4AAAB3Mw7hSAXm29svfZQxRhaEVL5x_7JEyWEywPtfIKmV2QMCZ61VfSvGxg-WCwS7OYnEonavdReiTgZ_3JalcPyInxYbHG668hbhfVRxCHWraC8lNhhhZvC45L32WDjkPRRy0qmoz_3SGzDDgumB2mgyjTHiqRdgEmopsEvouobSZDRxixXdANvTHyq85PmVnzKHK_-x7hVdYhu44J6P_oI4bZWnHA966Qna73q5YOPPevvZQVX8F71nVjDk4aJM5KhlAQwBDx5fzrV9Wk_R_Y-egz0sDL9oC3fBURGVwp4ywQ; HSID=AqgM3JlzrVA3Qkiyz; NID=43=F_oSZWyo9Niadk17m65QtM9alBJQ4YL0B0yAP71ruNqa5sV4JORImsQovU1PW2PEII7a-5KUi4YCRMCeytuhiwkWgSlWtHEAj_nt_EF8y84MNmrtmRzK9Kth96N4-bPZ'), ('frame.marked', '0'), ('eth.dst', '00:1a:8c:15:f9:80'), ('ssl.handshake.extensions_ec_point_format', ''), ('browser.server_type.sql', ''), ('http.chunk_boundary', ''), ('smb.file_attribute.archive', ''), ('smb.pid', ''), ('dns.id', ''), ('tcp.payload', '47:45:54:20:2f:63:6f:6d:70:6c:65:74:65:2f:73:65:61:72:63:68:3f:63:6c:69:65:6e:74:3d:63:68:72:6f:6d:65:26:68:6c:3d:65:6e:2d:55:53:26:71:3d:63:72:20:48:54:54:50:2f:31:2e:31:0d:0a:48:6f:73:74:3a:20:63:6c:69:65:6e:74:73:31:2e:67:6f:6f:67:6c:65:2e:63:61:0d:0a:43:6f:6e:6e:65:63:74:69:6f:6e:3a:20:6b:65:65:70:2d:61:6c:69:76:65:0d:0a:55:73:65:72:2d:41:67:65:6e:74:3a:20:4d:6f:7a:69:6c:6c:61:2f:35:2e:30:20:28:57:69:6e:64:6f:77:73:3b:20:55:3b:20:57:69:6e:64:6f:77:73:20:4e:54:20:36:2e:31:3b:20:65:6e:2d:55:53:29:20:41:70:70:6c:65:57:65:62:4b:69:74:2f:35:33:34:2e:31:30:20:28:4b:48:54:4d:4c:2c:20:6c:69:6b:65:20:47:65:63:6b:6f:29:20:43:68:72:6f:6d:65:2f:38:2e:30:2e:35:35:32:2e:32:33:37:20:53:61:66:61:72:69:2f:35:33:34:2e:31:30:0d:0a:41:63:63:65:70:74:2d:45:6e:63:6f:64:69:6e:67:3a:20:67:7a:69:70:2c:64:65:66:6c:61:74:65:2c:73:64:63:68:0d:0a:41:63:63:65:70:74:2d:4c:61:6e:67:75:61:67:65:3a:20:65:6e:2d:55:53:2c:65:6e:3b:71:3d:30:2e:38:0d:0a:41:63:63:65:70:74:2d:43:68:61:72:73:65:74:3a:20:49:53:4f:2d:38:38:35:39:2d:31:2c:75:74:66:2d:38:3b:71:3d:30:2e:37:2c:2a:3b:71:3d:30:2e:33:0d:0a:43:6f:6f:6b:69:65:3a:20:50:52:45:46:3d:49:44:3d:63:32:65:33:35:30:30:31:32:32:35:38:64:66:31:63:3a:55:3d:33:38:36:61:36:65:62:65:66:30:64:62:32:38:37:63:3a:46:46:3d:30:3a:54:4d:3d:31:32:39:34:31:36:34:32:39:34:3a:4c:4d:3d:31:32:39:34:31:36:34:32:39:34:3a:53:3d:62:63:75:77:4d:36:56:68:35:65:63:4b:78:71:6d:6b:3b:20:53:49:44:3d:44:51:41:41:41:4e:34:41:41:41:42:33:4d:77:37:68:53:41:58:6d:32:39:73:76:66:5a:51:78:52:68:61:45:56:4c:35:78:5f:37:4a:45:79:57:45:79:77:50:74:66:49:4b:6d:56:32:51:4d:43:5a:36:31:56:66:53:76:47:78:67:2d:57:43:77:53:37:4f:59:6e:45:6f:6e:61:76:64:52:65:69:54:67:5a:5f:33:4a:61:6c:63:50:79:49:6e:78:59:62:48:47:36:36:38:68:62:68:66:56:52:78:43:48:57:72:61:43:38:6c:4e:68:68:68:5a:76:43:34:35:4c:33:32:57:44:6a:6b:50:52:52:79:30:71:6d:6f:7a:5f:33:53:47:7a:44:44:67:75:6d:42:32:6d:67:79:6a:54:48:69:71:52:64:67:45:6d:6f:70:73:45:76:6f:75:6f:62:53:5a:44:52:78:69:78:58:64:41:4e:76:54:48:79:71:38:35:50:6d:56:6e:7a:4b:48:4b:5f:2d:78:37:68:56:64:59:68:75:34:34:4a:36:50:5f:6f:49:34:62:5a:57:6e:48:41:39:36:36:51:6e:61:37:33:71:35:59:4f:50:50:65:76:76:5a:51:56:58:38:46:37:31:6e:56:6a:44:6b:34:61:4a:4d:35:4b:68:6c:41:51:77:42:44:78:35:66:7a:72:56:39:57:6b:5f:52:5f:59:2d:65:67:7a:30:73:44:4c:39:6f:43:33:66:42:55:52:47:56:77:70:34:79:77:51:3b:20:48:53:49:44:3d:41:71:67:4d:33:4a:6c:7a:72:56:41:33:51:6b:69:79:7a:3b:20:4e:49:44:3d:34:33:3d:46:5f:6f:53:5a:57:79:6f:39:4e:69:61:64:6b:31:37:6d:36:35:51:74:4d:39:61:6c:42:4a:51:34:59:4c:30:42:30:79:41:50:37:31:72:75:4e:71:61:35:73:56:34:4a:4f:52:49:6d:73:51:6f:76:55:31:50:57:32:50:45:49:49:37:61:2d:35:4b:55:69:34:59:43:52:4d:43:65:79:74:75:68:69:77:6b:57:67:53:6c:57:74:48:45:41:6a:5f:6e:74:5f:45:46:38:79:38:34:4d:4e:6d:72:74:6d:52:7a:4b:39:4b:74:68:39:36:4e:34:2d:62:50:5a:0d:0a:0d:0a'), ('ssl.handshake.comp_methods_length', ''), ('tcp.reassembled.data', ''), ('udp.length', ''), ('browser.server_type.member', ''), ('smb.wct', ''), ('ssl.handshake.extensions_server_name_list_len', ''), ('ssl.record.version', ''), ('image-jfif.sos.se', ''), ('smb.flags.oplock', ''), ('smb.flags.receive_buffer', ''), ('dns.qry.type', ''), ('smb.nt.create_options.intermediate_buffering', ''), ('ssl.handshake.ciphersuite', ''), ('http.request.uri.path', '/complete/search'), ('smb.file_attribute.device', ''), ('icmp.resp_to', ''), ('rss.channel.image.url', ''), ('smb.access.read_ea', ''), ('browser.server_type.browser.potential', ''), ('tcp.window_size', '16288'), ('smb.access.write_ea', ''), ('smb.time', ''), ('tcp.analysis.window_full', ''), ('ssl.handshake.extension.data', ''), ('http.prev_response_in', ''), ('nbns.count.auth_rr', ''), ('image-jfif.sos.al', ''), ('dns.flags.response', ''), ('smb.po', ''), ('browser.server_type.server', ''), ('smb.file_attribute.directory', ''), ('png.chunk.flag.ancillary', ''), ('smb.flags.notify', ''), ('smb.file_attribute.temporary', ''), ('ssl.app_data', ''), ('dns.flags.tentative', ''), ('http.prev_request_in', ''), ('image-gif.image.left', ''), ('rss.channel.item', ''), ('png.chunk.len', ''), ('browser.os_major', ''), ('rss.channel.image', ''), ('png.srgb.intent', ''), ('xml.xmlpi.xml.encoding', ''), ('smb.share_access', ''), ('tcp.option_kind', ''), ('ssl.handshake.extensions_status_request_responder_ids_len', ''), ('tcp.analysis.push_bytes_sent', '943'), ('smb.file_attribute.compressed', ''), ('nbdgm.src.port', ''), ('xml.xmlpi.xml', ''), ('smb.file_attribute.not_content_indexed', ''), ('nbns.flags.opcode', ''), ('_score', ''), ('tcp.options.sack_le', ''), ('tcp.options.mss', ''), ('nbns.flags.recdesired', ''), ('arp.opcode', ''), ('udp.srcport', ''), ('bootp.hw.mac_addr', ''), ('nbdgm.type', ''), ('tcp.flags.str', 'Â·Â·Â·Â·Â·Â·Â·APÂ·Â·Â·'), ('smb.fid', ''), ('http.chat', ''), ('udp.checksum', ''), ('image-jfif.sos.ss', ''), ('eth.addr_resolved', '40:61:86:9a:f1:f5'), ('smb.rfid', ''), ('tcp.analysis.duplicate_ack', ''), ('image-gif.local.color_map.ordered', ''), ('image-jfif.entropy_coded_segment', ''), ('rss.channel.item.link', ''), ('tcp.hdr_len', '20'), ('snmp.value.null', ''), ('image-jfif.length', ''), ('tcp.analysis.initial_rtt', ''), ('ssl.handshake.extensions_npn', ''), ('ssl.handshake.challenge', ''), ('smb.nt.create_options.create_tree_connection', ''), ('smb.security.flags', ''), ('ip.dst', '72.14.213.138'), ('tcp.options.sack_perm', ''), ('ip.flags.rb', '0'), ('media.type', ''), ('smb.echo.count', ''), ('http.cookie_pair', 'NID=43=F_oSZWyo9Niadk17m65QtM9alBJQ4YL0B0yAP71ruNqa5sV4JORImsQovU1PW2PEII7a-5KUi4YCRMCeytuhiwkWgSlWtHEAj_nt_EF8y84MNmrtmRzK9Kth96N4-bPZ'), ('smb.nt.create_options.no_ea_knowledge', ''), ('icmp.code', ''), ('tcp.analysis.ack_lost_segment', ''), ('tcp.analysis.lost_segment', ''), ('browser.server_type.browser.backup', ''), ('smb.access.write_owner', ''), ('http.accept', ''), ('more testing', ''), ('tcp.analysis.acks_frame', ''), ('udp.stream', ''), ('eth.type', '0x00000800'), ('smb.tpc', ''), ('http.time', ''), ('smb.nt.create_options.random_access', ''), ('browser.period', ''), ('browser.server_type.xenix', ''), ('_index', 'packets-2020-04-07'), ('ssl.handshake.random_time', ''), ('dns.flags.recavail', ''), ('smb.nt.create_options.open_no_recall', ''), ('smb.alloc_size64', ''), ('image-gif.global.color_bpp', ''), ('smb.file_attribute.system', ''), ('ssl.handshake.challenge_length', ''), ('bootp.cookie', ''), ('smb.flags2.esn', ''), ('bootp.server', ''), ('smb.security.flags.effective_only', ''), ('png.chunk.crc', ''), ('tcp.flags.fin', '0'), ('nbns.class', ''), ('smb.error_code', ''), ('smb.nt.create_options.open_for_free_space_query', ''), ('smb.fid.closed_in', ''), ('eth.dst_resolved', '00:1a:8c:15:f9:80'), ('tcp.flags.push', '1'), ('_type', 'pcap_file'), ('http.request.method', 'GET'), ('smb.signature', ''), ('http.connection', 'keep-alive'), ('image-gif.extension.label', ''), ('image-jfif.sof.c_i', ''), ('png.ihdr.colour_type', ''), ('json.value.number', ''), ('tcp.options.nop', ''), ('nbns.type', ''), ('smb.flags2.ea', ''), ('arp.dst.hw_mac', ''), ('rtcp.psfb.fmt', ''), ('nbdgm.destination_name', ''), ('http.server', ''), ('smb.response_to', ''), ('dns.count.answers', ''), ('smb.access_mask', ''), ('smb.flags.caseless', ''), ('nbns.flags.truncated', ''), ('dns.response_to', ''), ('icmp.seq', ''), ('tcp.len', '943'), ('image-gif.global.color_map', ''), ('tcp.analysis.out_of_order', ''), ('http.request.line', 'Cookie: PREF=ID=c2e350012258df1c:U=386a6ebef0db287c:FF=0:TM=1294164294:LM=1294164294:S=bcuwM6Vh5ecKxqmk; SID=DQAAAN4AAAB3Mw7hSAXm29svfZQxRhaEVL5x_7JEyWEywPtfIKmV2QMCZ61VfSvGxg-WCwS7OYnEonavdReiTgZ_3JalcPyInxYbHG668hbhfVRxCHWraC8lNhhhZvC45L32WDjkPRRy0qmoz_3SGzDDgumB2mgyjTHiqRdgEmopsEvouobSZDRxixXdANvTHyq85PmVnzKHK_-x7hVdYhu44J6P_oI4bZWnHA966Qna73q5YOPPevvZQVX8F71nVjDk4aJM5KhlAQwBDx5fzrV9Wk_R_Y-egz0sDL9oC3fBURGVwp4ywQ; HSID=AqgM3JlzrVA3Qkiyz; NID=43=F_oSZWyo9Niadk17m65QtM9alBJQ4YL0B0yAP71ruNqa5sV4JORImsQovU1PW2PEII7a-5KUi4YCRMCeytuhiwkWgSlWtHEAj_nt_EF8y84MNmrtmRzK9Kth96N4-bPZ\\r\\n'), ('http.referer', ''), ('smb.last_write.time', ''), ('rss.channel.item.enclosure.type', ''), ('http.request.uri', '/complete/search?client=chrome&hl=en-US&q=cr'), ('rss.channel.copyright', ''), ('bootp.ip.server', ''), ('mailslot.priority', ''), ('http.last_modified', ''), ('I find the term packet sniffer dirty.', ''), ('bootp.dhcp', ''), ('dns.count.labels', ''), ('smb.flags2.dfs', ''), ('bootp.option.end', ''), ('tcp.segment_data', ''), ('rtcp.version', ''), ('urlencoded-form.key', ''), ('udp.possible_traceroute', ''), ('nbdgm.pkt_offset', ''), ('ip.host', '72.14.213.138'), ('bootp.ip.client', ''), ('png.chunk.flag.private', ''), ('dns.time', ''), ('smb.create_flags', ''), ('frame.len', '997'), ('rss.channel.item.title', ''), ('frame.protocols', 'eth:ethertype:ip:tcp:http'), ('arp.src.hw_mac', ''), ('dns.flags.authenticated', ''), ('dns.cname', ''), ('tcp.options.mss_val', ''), ('png.ihdr.filter_method', ''), ('icmp.checksum.status', ''), ('tcp.flags.ack', '1'), ('http.host', 'clients1.google.ca'), ('_ws.expert.severity', '2097152'), ('bootp.file', ''), ('bootp.secs', ''), ('snmp.name', ''), ('dns.flags.checkdisable', ''), ('nbdgm.dgram_len', ''), ('smb.error_class', ''), ('smb.tdc', ''), ('icmp.type', ''), ('http.response.line', ''), ('rtcp.length', ''), ('ssl.record.length', ''), ('tcp.time_delta', '0.000000000'), ('icmp.checksum', ''), ('dns.resp.ttl', ''), ('ip.ttl', '128'), ('ssl.handshake.length', ''), ('png.ihdr.width', ''), ('smb.share.access.write', ''), ('http.subdissector_failed', ''), ('png.chunk.flag.stc', ''), ('png.ihdr.interlace_method', ''), ('nat-pmp.external_port', ''), ('smb.data_offset', ''), ('smb.access.write_dac', ''), ('image-gif.global.color_map.ordered', ''), ('rss.channel.item.guid.ispermalink', ''), ('ip.checksum', '0x00009e7c'), ('smb.bcc', ''), ('xml.unknown', ''), ('frame.time_epoch', '1295981542.484409000'), ('http.date', ''), ('png.chunk.type', ''), ('smb.mdc', ''), ('smb.access.delete', ''), ('http.accept_encoding', 'gzip,deflate,sdch'), ('smb.dc', ''), ('data.len', ''), ('tcp.flags.urg', '0'), ('smb.transaction.flags.dtid', ''), ('rtcp.padding', ''), ('browser.server_type.ntw', ''), ('smb.nt.create_options.backup_intent', ''), ('tcp.options.sack', ''), ('tcp.options', ''), ('smb.file_attribute.offline', ''), ('smb.create.disposition', ''), ('smb.flags2', ''), ('tcp.options.wscale.multiplier', ''), ('tcp.analysis.rto', ''), ('browser.server_type.print', ''), ('browser.server_type.nts', ''), ('image-gif.image.code_size', ''), ('ssl.record.content_type', ''), ('smb.file_attribute.normal', ''), ('rss.channel.item.guid', ''), ('smb.fid.opened_in', ''), ('http.request.full_uri', 'http://clients1.google.ca/complete/search?client=chrome&hl=en-US&q=cr'), ('rss.channel.item.pubdate', ''), ('rss.channel.item.category', ''), ('bootp.hw.len', ''), ('smb.nt_status', ''), ('dns.flags.truncated', ''), ('ssl.alert_message', ''), ('nbns.id', ''), ('browser.server_type.novell', ''), ('http.response.phrase', ''), ('image-jfif.sof.v_i', ''), ('udp.dstport', ''), ('tcp.options.wscale', ''), ('nbss.type', ''), ('nbdgm.dgram_id', ''), ('image-jfif.sof.h_i', ''), ('smb.file_attribute', ''), ('smb.file_attribute.sparse', ''), ('bootp.flags.reserved', ''), ('smb.share.access.read', ''), ('png.phys.unit', ''), ('http.request.version', 'HTTP/1.1'), ('dns.ns', ''), ('tcp.options.sack.count', ''), ('eth.addr', '40:61:86:9a:f1:f5'), ('tcp.options.wscale.shift', ''), ('snmp.request_id', '')])\n"
     ]
    }
   ],
   "source": [
    "# the output is a gzipped csv, so let's quickly pop that open and see what we have\n",
    "# we're going to use DictReader from the CSV lib, so we'll get back a list of dictionaries, where the keys in the dicts are the fieldnames\n",
    "# each dictionary in the list is a record (packet)\n",
    "\n",
    "import csv\n",
    "from networkml.featurizers.csv_to_features import CSVToFeatures\n",
    "instance = CSVToFeatures()\n",
    "rows = list(csv.DictReader(instance.get_reader('trace_ab12_2001-01-01_02_03-client-ip-1-2-3-4.pcap.csv.gz', True)))\n",
    "print(rows[0])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:networkml.featurizers.csv_to_features:Importing trace_ab12_2001-01-01_02_03-client-ip-1-2-3-4.pcap.csv.gz size 15691804\n",
      "INFO:networkml.featurizers.csv_to_features:Featurizing trace_ab12_2001-01-01_02_03-client-ip-1-2-3-4.pcap.csv.gz\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing class: Generic\n",
      "Importing class: Host\n",
      "Importing class: SessionHost\n",
      "Importing class: Packet\n",
      "Importing class: Flow\n",
      "running Host/host_tshark_all...calculating intermediates.9 MACs, 1 sessions.MAC 00:04:00:81:81:d0 has minimum number of source IPs, selected as canonical source.MAC 1/1 100%."
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:networkml.featurizers.csv_to_features:Finished trace_ab12_2001-01-01_02_03-client-ip-1-2-3-4.pcap.csv.gz. 1/1 CSVs done.\n",
      "INFO:networkml.featurizers.csv_to_features:GZipped CSV file(s) written out to: ['trace_ab12_2001-01-01_02_03-client-ip-1-2-3-4.features.csv.gz']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "''"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# we have a gzipped csv with all of the fields we could extract at the 'packet' level\n",
    "# (we could have supplied an arg above to do a different level, like 'flow')\n",
    "# we can now take that file and reduce or change or add which fields should be included using the featurizer\n",
    "\n",
    "# let's set a path to a gzipped csv (we'll use one we just made)\n",
    "path = 'trace_ab12_2001-01-01_02_03-client-ip-1-2-3-4.pcap.csv.gz'\n",
    "\n",
    "# let's change the output so it's easy to find\n",
    "output = 'trace_ab12_2001-01-01_02_03-client-ip-1-2-3-4.features.csv.gz'\n",
    "\n",
    "# we need to specify where the featurizer functions are\n",
    "features_path = '../networkml/featurizers/funcs'\n",
    "\n",
    "import sys\n",
    "# set arguments for arg parse\n",
    "sys.argv = ['csv_to_features.py', f'-o{output}', f'-p{features_path}', path]\n",
    "\n",
    "from networkml.featurizers.csv_to_features import CSVToFeatures\n",
    "instance = CSVToFeatures()\n",
    "instance.main()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OrderedDict([('host_key', '00:04:00:81:81:d0'), ('tshark_max_frame_len_out', '124'), ('tshark_min_frame_len_out', '124'), ('tshark_count_frame_len_out', '8'), ('tshark_total_frame_len_out', '992'), ('tshark_average_frame_len_out', '124.0'), ('tshark_median_frame_len_out', '124.0'), ('tshark_variance_frame_len_out', '0.0'), ('tshark_25q_frame_len_out', '124.0'), ('tshark_75q_frame_len_out', '124.0'), ('tshark_max_time_delta_out', '0.059905'), ('tshark_min_time_delta_out', '0.001216'), ('tshark_count_time_delta_out', '8'), ('tshark_total_time_delta_out', '0.301913'), ('tshark_average_time_delta_out', '0.037739125'), ('tshark_median_time_delta_out', '0.041939500000000005'), ('tshark_variance_time_delta_out', '0.0005967626078392858'), ('tshark_25q_time_delta_out', '0.02140825'), ('tshark_75q_time_delta_out', '0.0594175'), ('tshark_ip_dsfield_ecn0_out', '0'), ('tshark_ip_dsfield_ecn1_out', '0'), ('tshark_ip_dsfield_dscp0_out', '0'), ('tshark_ip_dsfield_dscp1_out', '0'), ('tshark_ip_dsfield_dscp2_out', '0'), ('tshark_ip_dsfield_dscp3_out', '0'), ('tshark_ip_dsfield_dscp4_out', '0'), ('tshark_ip_dsfield_dscp5_out', '0'), ('tshark_ip_flags_fin_out', '0'), ('tshark_ip_flags_syn_out', '0'), ('tshark_ip_flags_rst_out', '0'), ('tshark_ip_flags_psh_out', '0'), ('tshark_ip_flags_ack_out', '0'), ('tshark_ip_flags_urg_out', '0'), ('tshark_ip_flags_ece_out', '0'), ('tshark_ip_flags_cwr_out', '0'), ('tshark_ip_flags_ns_out', '0'), ('tshark_tcp_flags_fin_out', '0'), ('tshark_tcp_flags_syn_out', '0'), ('tshark_tcp_flags_rst_out', '0'), ('tshark_tcp_flags_psh_out', '0'), ('tshark_tcp_flags_ack_out', '0'), ('tshark_tcp_flags_urg_out', '0'), ('tshark_tcp_flags_ece_out', '0'), ('tshark_tcp_flags_cwr_out', '0'), ('tshark_tcp_flags_ns_out', '0'), ('tshark_udp_priv_port_161_out', '1'), ('tshark_udp_priv_port_67_out', '0'), ('tshark_udp_priv_port_68_out', '0'), ('tshark_udp_priv_port_69_out', '0'), ('tshark_udp_priv_port_631_out', '0'), ('tshark_udp_priv_port_137_out', '0'), ('tshark_udp_priv_port_138_out', '0'), ('tshark_udp_priv_port_139_out', '0'), ('tshark_udp_priv_port_110_out', '0'), ('tshark_udp_priv_port_143_out', '0'), ('tshark_udp_priv_port_80_out', '0'), ('tshark_udp_priv_port_53_out', '0'), ('tshark_udp_priv_port_22_out', '0'), ('tshark_udp_priv_port_23_out', '0'), ('tshark_udp_priv_port_88_out', '0'), ('tshark_udp_priv_port_25_out', '0'), ('tshark_udp_priv_port_443_out', '0'), ('tshark_udp_priv_port_123_out', '0'), ('tshark_udp_priv_port_other_out', '0'), ('tshark_udp_nonpriv_port_5349_out', '0'), ('tshark_udp_nonpriv_port_5222_out', '0'), ('tshark_udp_nonpriv_port_2375_out', '0'), ('tshark_udp_nonpriv_port_2376_out', '0'), ('tshark_udp_nonpriv_port_5353_out', '0'), ('tshark_udp_nonpriv_port_5354_out', '0'), ('tshark_udp_nonpriv_port_1900_out', '0'), ('tshark_udp_nonpriv_port_5357_out', '0'), ('tshark_udp_nonpriv_port_6653_out', '0'), ('tshark_udp_nonpriv_port_other_out', '0'), ('tshark_tcp_priv_port_161_out', '0'), ('tshark_tcp_priv_port_67_out', '0'), ('tshark_tcp_priv_port_68_out', '0'), ('tshark_tcp_priv_port_69_out', '0'), ('tshark_tcp_priv_port_631_out', '0'), ('tshark_tcp_priv_port_137_out', '0'), ('tshark_tcp_priv_port_138_out', '0'), ('tshark_tcp_priv_port_139_out', '0'), ('tshark_tcp_priv_port_110_out', '0'), ('tshark_tcp_priv_port_143_out', '0'), ('tshark_tcp_priv_port_80_out', '0'), ('tshark_tcp_priv_port_53_out', '0'), ('tshark_tcp_priv_port_22_out', '0'), ('tshark_tcp_priv_port_23_out', '0'), ('tshark_tcp_priv_port_88_out', '0'), ('tshark_tcp_priv_port_25_out', '0'), ('tshark_tcp_priv_port_443_out', '0'), ('tshark_tcp_priv_port_123_out', '0'), ('tshark_tcp_priv_port_other_out', '0'), ('tshark_tcp_nonpriv_port_5349_out', '0'), ('tshark_tcp_nonpriv_port_5222_out', '0'), ('tshark_tcp_nonpriv_port_2375_out', '0'), ('tshark_tcp_nonpriv_port_2376_out', '0'), ('tshark_tcp_nonpriv_port_5353_out', '0'), ('tshark_tcp_nonpriv_port_5354_out', '0'), ('tshark_tcp_nonpriv_port_1900_out', '0'), ('tshark_tcp_nonpriv_port_5357_out', '0'), ('tshark_tcp_nonpriv_port_6653_out', '0'), ('tshark_tcp_nonpriv_port_other_out', '0'), ('tshark_max_frame_len_in', '120'), ('tshark_min_frame_len_in', '120'), ('tshark_count_frame_len_in', '8'), ('tshark_total_frame_len_in', '960'), ('tshark_average_frame_len_in', '120.0'), ('tshark_median_frame_len_in', '120.0'), ('tshark_variance_frame_len_in', '0.0'), ('tshark_25q_frame_len_in', '120.0'), ('tshark_75q_frame_len_in', '120.0'), ('tshark_max_time_delta_in', '0.4722'), ('tshark_min_time_delta_in', '0.001993'), ('tshark_count_time_delta_in', '8'), ('tshark_total_time_delta_in', '1.4286780000000001'), ('tshark_average_time_delta_in', '0.17858475000000001'), ('tshark_median_time_delta_in', '0.14042'), ('tshark_variance_time_delta_in', '0.023220977560785717'), ('tshark_25q_time_delta_in', '0.10236624999999999'), ('tshark_75q_time_delta_in', '0.254556'), ('tshark_ip_dsfield_ecn0_in', '0'), ('tshark_ip_dsfield_ecn1_in', '0'), ('tshark_ip_dsfield_dscp0_in', '0'), ('tshark_ip_dsfield_dscp1_in', '0'), ('tshark_ip_dsfield_dscp2_in', '0'), ('tshark_ip_dsfield_dscp3_in', '0'), ('tshark_ip_dsfield_dscp4_in', '0'), ('tshark_ip_dsfield_dscp5_in', '0'), ('tshark_ip_flags_fin_in', '0'), ('tshark_ip_flags_syn_in', '0'), ('tshark_ip_flags_rst_in', '0'), ('tshark_ip_flags_psh_in', '0'), ('tshark_ip_flags_ack_in', '0'), ('tshark_ip_flags_urg_in', '0'), ('tshark_ip_flags_ece_in', '0'), ('tshark_ip_flags_cwr_in', '0'), ('tshark_ip_flags_ns_in', '0'), ('tshark_tcp_flags_fin_in', '0'), ('tshark_tcp_flags_syn_in', '0'), ('tshark_tcp_flags_rst_in', '0'), ('tshark_tcp_flags_psh_in', '0'), ('tshark_tcp_flags_ack_in', '0'), ('tshark_tcp_flags_urg_in', '0'), ('tshark_tcp_flags_ece_in', '0'), ('tshark_tcp_flags_cwr_in', '0'), ('tshark_tcp_flags_ns_in', '0'), ('tshark_udp_priv_port_161_in', '1'), ('tshark_udp_priv_port_67_in', '0'), ('tshark_udp_priv_port_68_in', '0'), ('tshark_udp_priv_port_69_in', '0'), ('tshark_udp_priv_port_631_in', '0'), ('tshark_udp_priv_port_137_in', '0'), ('tshark_udp_priv_port_138_in', '0'), ('tshark_udp_priv_port_139_in', '0'), ('tshark_udp_priv_port_110_in', '0'), ('tshark_udp_priv_port_143_in', '0'), ('tshark_udp_priv_port_80_in', '0'), ('tshark_udp_priv_port_53_in', '0'), ('tshark_udp_priv_port_22_in', '0'), ('tshark_udp_priv_port_23_in', '0'), ('tshark_udp_priv_port_88_in', '0'), ('tshark_udp_priv_port_25_in', '0'), ('tshark_udp_priv_port_443_in', '0'), ('tshark_udp_priv_port_123_in', '0'), ('tshark_udp_priv_port_other_in', '0'), ('tshark_udp_nonpriv_port_5349_in', '0'), ('tshark_udp_nonpriv_port_5222_in', '0'), ('tshark_udp_nonpriv_port_2375_in', '0'), ('tshark_udp_nonpriv_port_2376_in', '0'), ('tshark_udp_nonpriv_port_5353_in', '0'), ('tshark_udp_nonpriv_port_5354_in', '0'), ('tshark_udp_nonpriv_port_1900_in', '0'), ('tshark_udp_nonpriv_port_5357_in', '0'), ('tshark_udp_nonpriv_port_6653_in', '0'), ('tshark_udp_nonpriv_port_other_in', '0'), ('tshark_tcp_priv_port_161_in', '0'), ('tshark_tcp_priv_port_67_in', '0'), ('tshark_tcp_priv_port_68_in', '0'), ('tshark_tcp_priv_port_69_in', '0'), ('tshark_tcp_priv_port_631_in', '0'), ('tshark_tcp_priv_port_137_in', '0'), ('tshark_tcp_priv_port_138_in', '0'), ('tshark_tcp_priv_port_139_in', '0'), ('tshark_tcp_priv_port_110_in', '0'), ('tshark_tcp_priv_port_143_in', '0'), ('tshark_tcp_priv_port_80_in', '0'), ('tshark_tcp_priv_port_53_in', '0'), ('tshark_tcp_priv_port_22_in', '0'), ('tshark_tcp_priv_port_23_in', '0'), ('tshark_tcp_priv_port_88_in', '0'), ('tshark_tcp_priv_port_25_in', '0'), ('tshark_tcp_priv_port_443_in', '0'), ('tshark_tcp_priv_port_123_in', '0'), ('tshark_tcp_priv_port_other_in', '0'), ('tshark_tcp_nonpriv_port_5349_in', '0'), ('tshark_tcp_nonpriv_port_5222_in', '0'), ('tshark_tcp_nonpriv_port_2375_in', '0'), ('tshark_tcp_nonpriv_port_2376_in', '0'), ('tshark_tcp_nonpriv_port_5353_in', '0'), ('tshark_tcp_nonpriv_port_5354_in', '0'), ('tshark_tcp_nonpriv_port_1900_in', '0'), ('tshark_tcp_nonpriv_port_5357_in', '0'), ('tshark_tcp_nonpriv_port_6653_in', '0'), ('tshark_tcp_nonpriv_port_other_in', '0'), ('tshark_ipv4', '1'), ('tshark_ipv6', '0'), ('tshark_ipx', '0'), ('tshark_nonip', '0'), ('tshark_both_private_ip', '1'), ('tshark_ipv4_multicast', '0'), ('tshark_wk_ip_protocol_tcp', '0'), ('tshark_wk_ip_protocol_udp', '1'), ('tshark_wk_ip_protocol_icmp', '0'), ('tshark_wk_ip_protocol_arp', '0'), ('tshark_wk_ip_protocol_icmpv6', '0'), ('tshark_wk_ip_protocol_gre', '0'), ('tshark_wk_ip_protocol_esp', '0'), ('tshark_wk_ip_protocol_ah', '0'), ('tshark_tagged_vlan', '0'), ('tshark_unique_srcips', '1'), ('tshark_unique_dstips', '1')])\n"
     ]
    }
   ],
   "source": [
    "# the output is a gzipped csv again, so let's quickly pop that open and see what we have\n",
    "# we're going to use DictReader from the CSV lib, so we'll get back a list of dictionaries, where the keys in the dicts are the fieldnames\n",
    "# each dictionary in the list is a record (packet)\n",
    "\n",
    "from networkml.featurizers.csv_to_features import CSVToFeatures\n",
    "instance = CSVToFeatures()\n",
    "rows = list(csv.DictReader(instance.get_reader('trace_ab12_2001-01-01_02_03-client-ip-1-2-3-4.features.csv.gz', True)))\n",
    "print(rows[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Writing New Featuzier Functions\n",
    "now that we can see how to turn PCAPs into CSVs and then use the featurizer to change the columns and make a new CSV\n",
    "let's now look at how to write new featurizer functions and use them here's an example of an already included class with functions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "from networkml.featurizers.features import Features\n",
      "\n",
      "\n",
      "\n",
      "class Flow(Features):\n",
      "\n",
      "\n",
      "\n",
      "    def default_tcp_5tuple(self, rows):\n",
      "\n",
      "        fields = ['ip.src_host', 'ip.dst_host', 'tcp.dstport', 'tcp.srcport', 'frame.protocols']\n",
      "\n",
      "        return self.get_columns(fields, rows)\n",
      "\n",
      "\n",
      "\n",
      "    def default_udp_5tuple(self, rows):\n",
      "\n",
      "        fields = ['ip.src_host', 'ip.dst_host', 'udp.dstport', 'udp.srcport', 'frame.protocols']\n",
      "\n",
      "        return self.get_columns(fields, rows)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "with open('../networkml/featurizers/funcs/flow.py', 'r') as f:\n",
    "    for line in f:\n",
    "        print(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OrderedDict([('ip.id', '0x000076e0'), ('dns.qry.name.len', ''), ('smb.access.delete_child', ''), ('smb.access.read_control', ''), ('dns.resp.len', ''), ('smb.nt.create_options.non_directory', ''), ('smb.pc', ''), ('bootp.option.type', ''), ('ssl.handshake.extensions_status_request_exts_len', ''), ('tcp.window_size_scalefactor', '-1'), ('tcp.connection.fin', ''), ('nbdgm.src.ip', ''), ('ip.flags', '0x00004000'), ('smb.nt.create_options.reserve_opfilter', ''), ('eth.lg', '0'), ('http.request.uri.query.parameter', 'q=cr'), ('http.chunk_size', ''), ('rss.channel.image.title', ''), ('smb.pid.high', ''), ('nbns.count.answers', ''), ('ssl.handshake.extensions_status_request_type', ''), ('frame.time_relative', '0.000000000'), ('tcp.checksum', '0x00005df8'), ('ssl.handshake.version', ''), ('snmp.error_index', ''), ('ssl.handshake.extensions_ec_point_formats_length', ''), ('png.phys.vert', ''), ('browser.server_type.dialin', ''), ('dns.retransmit_request', ''), ('frame.cap_len', '997'), ('smb.timeout', ''), ('arp.src.proto_ipv4', ''), ('rss.version', ''), ('smb.nt.create_options.sync_io_nonalert', ''), ('smb.nt.create_options.no_compression', ''), ('arp.hw.type', ''), ('nat-pmp.opcode', ''), ('ssl.handshake.extensions_reneg_info_len', ''), ('smb.security.flags.context_tracking', ''), ('ssl.handshake.random', ''), ('ip.src', '192.168.3.131'), ('ssl.handshake.random_bytes', ''), ('http.set_cookie', ''), ('xml.tag', ''), ('browser.proto_minor', ''), ('rss.channel.link', ''), ('ip.flags.mf', '0'), ('image-jfif.sos.ac_entropy_selector', ''), ('dns.resp.type', ''), ('ip.dsfield.dscp', '0'), ('ssl.handshake.cipher_suites_length', ''), ('nbns.count.queries', ''), ('ssl.handshake.type', ''), ('tcp.option_len', ''), ('tcp.analysis.rto_frame', ''), ('dns.count.auth_rr', ''), ('http.file_data', ''), ('http.transfer_encoding', ''), ('smb.access.system_security', ''), ('tcp.time_relative', '0.000000000'), ('tcp.port', '80'), ('http.request_in', ''), ('browser.command', ''), ('tcp.connection.rst', ''), ('arp.dst.proto_ipv4', ''), ('udp.checksum.status', ''), ('ssl.handshake', ''), ('icmp.seq_le', ''), ('ssl.handshake.extensions_server_name', ''), ('smb.andxoffset', ''), ('rss.channel.item.enclosure.length', ''), ('xml.xmlpi.xml.version', ''), ('ip.checksum.status', '2'), ('rss.channel.language', ''), ('tcp.srcport', '57011'), ('image-gif.image.height', ''), ('smb.flags.lock', ''), ('smb.nt.create_options.open_by_fileid', ''), ('bootp.flags', ''), ('http.response.code.desc', ''), ('smb.echo.data', ''), ('ip.dst_host', '72.14.213.138'), ('tcp.window_size_value', '16288'), ('image-gif.global.color_map.present', ''), ('smb.cmd', ''), ('smb.nt.create_options.write_through', ''), ('_ws.malformed', ''), ('rss.channel.item.description', ''), ('smb.flags2.nt_error', ''), ('nbns.count.add_rr', ''), ('tcp.stream', '0'), ('image-jfif.sos.component_selector', ''), ('browser.server_type', ''), ('ssl.handshake.extension.len', ''), ('_ws.expert.message', 'GET /complete/search?client=chrome&hl=en-US&q=cr HTTP/1.1\\\\r\\\\n'), ('data.data', ''), ('smb.mid', ''), ('smb.flags2.compressed', ''), ('smb.trans_name', ''), ('nbdgm.source_name', ''), ('tcp.flags.ns', '0'), ('arp.proto.type', ''), ('dns.flags.authoritative', ''), ('eth.src_resolved', '40:61:86:9a:f1:f5'), ('http.response_number', ''), ('tcp.analysis.ack_rtt', ''), ('ip.ttl.too_small', ''), ('tcp.dstport', '80'), ('mailslot.opcode', ''), ('data-text-lines', ''), ('bootp.type', ''), ('dns.count.add_rr', ''), ('tcp.flags.cwr', '0'), ('tcp.analysis.bytes_in_flight', '943'), ('ip.dsfield.ecn', '0'), ('smb.share.access.delete', ''), ('nbns.name', ''), ('browser.server_type.apple', ''), ('smb.file_attribute.reparse', ''), ('ssl.handshake.extension.type', ''), ('_ws.malformed.expert', ''), ('bootp.option.padding', ''), ('smb.file', ''), ('ssl.handshake.session_id_length', ''), ('ip.proto', '6'), ('tcp.analysis.duplicate_ack_frame', ''), ('http.response_for.uri', ''), ('rss.channel', ''), ('smb.file_attribute.encrypted', ''), ('ssl.handshake.cipher_spec_len', ''), ('browser.server_type.domain_controller', ''), ('http.request_number', '1'), ('browser.server', ''), ('png.chunk.data', ''), ('dns.flags.opcode', ''), ('icmp.unused', ''), ('image-gif.image.top', ''), ('arp.proto.size', ''), ('image-gif.image.width', ''), ('bootp.ip.your', ''), ('ssl.handshake.cipherspec', ''), ('image-gif.image_background_index', ''), ('browser.server_type.wfw', ''), ('frame.time_delta_displayed', '0.000000000'), ('ssl.handshake.extensions_supported_group', ''), ('smb.flags2.reparse_path', ''), ('bootp.hops', ''), ('smb.msc', ''), ('smb.flags2.long_names_used', ''), ('image-jfif.remain_seg_data', ''), ('smb.access.maximum_allowed', ''), ('dns.flags', ''), ('icmp.ident', ''), ('tcp.options.eol', ''), ('nbss.flags.e', ''), ('bootp.flags.bc', ''), ('image-jfif.sof.precision', ''), ('png.ihdr.compression_method', ''), ('snmp.error_status', ''), ('rss.channel.description', ''), ('ip.hdr_len', '20'), ('ip.len', '983'), ('snmp.variable_bindings', ''), ('browser.server_type.osf', ''), ('image-gif.screen.height', ''), ('bootp.hw.type', ''), ('smb.server_component', ''), ('frame.encap_type', '1'), ('tcp.reassembled.length', ''), ('frame.number', '1'), ('bootp.hw.addr_padding', ''), ('smb.create_options', ''), ('nat-pmp.reserved', ''), ('nbns.flags.broadcast', ''), ('tcp.seq', '1'), ('nbdgm.flags', ''), ('image-gif.global.bpp', ''), ('smb.access.synchronize', ''), ('tcp.analysis.window_update', ''), ('arp.hw.size', ''), ('smb.transaction.flags.owt', ''), ('tcp.flags', '0x00000018'), ('browser.server_type.w95', ''), ('nat-pmp.version', ''), ('tcp.flags.ecn', '0'), ('json.key', ''), ('png.ihdr.bitdepth', ''), ('http.request', '1'), ('http.content_encoding', ''), ('mailslot.name', ''), ('smb.sc', ''), ('ssl.ignored_unknown_record', ''), ('tcp.segment', ''), ('browser.server_type.browser.master', ''), ('frame.ignored', '0'), ('http.cache_control', ''), ('dns.resp.class', ''), ('rss.channel.lastbuilddate', ''), ('image-jfif.sof.nf', ''), ('image-gif.version', ''), ('browser.sig', ''), ('bootp.id', ''), ('smb.flags2.long_names_allowed', ''), ('frame.offset_shift', '0.000000000'), ('image-gif.local.color_bpp', ''), ('tcp.flags.syn', '0'), ('udp.port', ''), ('ssl.handshake.extensions_server_name_type', ''), ('icmp.resptime', ''), ('smb.access.generic_all', ''), ('smb.nt.create_options.eight_dot_three_only', ''), ('smb.nt.create.oplock', ''), ('dns.flags.recdesired', ''), ('http.response.version', ''), ('mailslot.size', ''), ('tcp.ack.nonzero', ''), ('rtcp.pt', ''), ('dns.qry.name', ''), ('nat-pmp.internal_port', ''), ('nbdgm.next', ''), ('browser.server_type.workstation', ''), ('png.gama.gamma', ''), ('image-jfif.sos.dc_entropy_selector', ''), ('mailslot.class', ''), ('nbdgm.node_type', ''), ('browser.proto_major', ''), ('image-gif.data_block', ''), ('tcp.nxtseq', '944'), ('http.user_agent', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.237 Safari/534.10'), ('smb.access.write_attributes', ''), ('dns.retransmit_request_in', ''), ('rss.channel.item.enclosure.url', ''), ('image-jfif.marker', ''), ('nbss.length', ''), ('smb.impersonation.level', ''), ('bootp.ip.relay', ''), ('ip.addr', '72.14.213.138'), ('smb.access.generic_write', ''), ('smb.flags2.roe', ''), ('ssl.handshake.extensions_server_name_len', ''), ('smb.access.read_attributes', ''), ('smb.nt.create_options.sequential_only', ''), ('browser.server_type.local', ''), ('tcp.segment.count', ''), ('smb.flags2.sec_sig', ''), ('browser.os_minor', ''), ('smb.nt.create_options.sync_io_alert', ''), ('tcp.connection.sack', ''), ('eth.ig', '0'), ('rss.channel.title', ''), ('dns.flags.rcode', ''), ('browser.server_type.browser.domain_master', ''), ('smb.flags2.sec_sig_required', ''), ('snmp.data', ''), ('image-jfif.sof.lines', ''), ('http.content_length', ''), ('image-jfif.sos.ns', ''), ('nbns.flags', ''), ('snmp.community', ''), ('browser.server_type.dfs', ''), ('dns.flags.conflict', ''), ('image-jfif.sos.ah', ''), ('ssl.handshake.extensions_npn_str_len', ''), ('smb.file_name_len', ''), ('smb.tid', ''), ('nbns.flags.response', ''), ('dns.qry.class', ''), ('browser.server_type.backup_controller', ''), ('ssl.handshake.session_id', ''), ('png.ihdr.height', ''), ('image-gif.screen.width', ''), ('snmp.value.octets', ''), ('rss.channel.image.link', ''), ('rss.channel.ttl', ''), ('tcp.checksum.status', '2'), ('browser.comment', ''), ('smb.echo.seq_num', ''), ('ssl.handshake.comp_method', ''), ('image-gif.end', ''), ('image-jfif.sof.samples_per_line', ''), ('dns.flags.z', ''), ('eth.src', '40:61:86:9a:f1:f5'), ('browser.update_count', ''), ('http.response', ''), ('smb.access.write', ''), ('xml.attribute', ''), ('rtcp.mediassrc', ''), ('tcp.flags.reset', '0'), ('dns.resp.name', ''), ('smb.flags', ''), ('frame.time', 'Jan 26, 2011 07:52:22.484409000 NZDT'), ('urlencoded-form.value', ''), ('browser.server_type.domainenum', ''), ('snmp.value.int', ''), ('frame.time_delta', '0.000000000'), ('smb.nt.create_options.open_reparse_point', ''), ('image-jfif.sof.tq_i', ''), ('smb.file_attribute.hidden', ''), ('ssl', ''), ('http.content_type', ''), ('ssl.handshake.extensions_length', ''), ('dns.a', ''), ('dns.retransmission', ''), ('_ws.expert.group', '33554432'), ('smb.flags.canon', ''), ('browser.windows_version', ''), ('smb.transaction.flags', ''), ('tcp.connection.syn', ''), ('http.accept_language', 'en-US,en;q=0.8'), ('tcp.analysis.retransmission', ''), ('smb.mpc', ''), ('tcp.urgent_pointer', '0'), ('http.request.uri.query', 'client=chrome&hl=en-US&q=cr'), ('smb.nt.create_options.complete_if_oplocked', ''), ('http.response.code', ''), ('browser.server_type.vms', ''), ('smb.nt.create_options.directory', ''), ('smb.flags2.string', ''), ('png.signature', ''), ('tcp.analysis.duplicate_ack_num', ''), ('smb.access.append', ''), ('ip.dsfield', '0x00000000'), ('http.content_length_header', ''), ('ssl.handshake.extensions_supported_groups_length', ''), ('rtcp.senderssrc', ''), ('rss.channel.item.enclosure', ''), ('smb.file_attribute.read_only', ''), ('smb.access.generic_read', ''), ('xml.cdata', ''), ('ip.frag_offset', '0'), ('smb.reserved', ''), ('nbdgm.first', ''), ('smb.access.generic_execute', ''), ('smb.nt.create.batch_oplock', ''), ('smb.file_attribute.volume', ''), ('http.location', ''), ('smb.nt.create.dir', ''), ('image-gif.local.color_map.present', ''), ('smb.nt.create_options.delete_on_close', ''), ('smb.flags.response', ''), ('ip.flags.df', '1'), ('smb.uid', ''), ('ip.version', '4'), ('dns.count.queries', ''), ('smb.access.execute', ''), ('smb.nt.create.ext', ''), ('eth.padding', ''), ('tcp.ack', '1'), ('ip.src_host', '192.168.3.131'), ('nbss.flags', ''), ('tcp.options.sack_re', ''), ('nat-pmp.pml', ''), ('tcp.flags.res', '0'), ('png.phys.horiz', ''), ('snmp.version', ''), ('browser.server_type.time', ''), ('smb.access.read', ''), ('http.cookie', 'PREF=ID=c2e350012258df1c:U=386a6ebef0db287c:FF=0:TM=1294164294:LM=1294164294:S=bcuwM6Vh5ecKxqmk; SID=DQAAAN4AAAB3Mw7hSAXm29svfZQxRhaEVL5x_7JEyWEywPtfIKmV2QMCZ61VfSvGxg-WCwS7OYnEonavdReiTgZ_3JalcPyInxYbHG668hbhfVRxCHWraC8lNhhhZvC45L32WDjkPRRy0qmoz_3SGzDDgumB2mgyjTHiqRdgEmopsEvouobSZDRxixXdANvTHyq85PmVnzKHK_-x7hVdYhu44J6P_oI4bZWnHA966Qna73q5YOPPevvZQVX8F71nVjDk4aJM5KhlAQwBDx5fzrV9Wk_R_Y-egz0sDL9oC3fBURGVwp4ywQ; HSID=AqgM3JlzrVA3Qkiyz; NID=43=F_oSZWyo9Niadk17m65QtM9alBJQ4YL0B0yAP71ruNqa5sV4JORImsQovU1PW2PEII7a-5KUi4YCRMCeytuhiwkWgSlWtHEAj_nt_EF8y84MNmrtmRzK9Kth96N4-bPZ'), ('frame.marked', '0'), ('eth.dst', '00:1a:8c:15:f9:80'), ('ssl.handshake.extensions_ec_point_format', ''), ('browser.server_type.sql', ''), ('http.chunk_boundary', ''), ('smb.file_attribute.archive', ''), ('smb.pid', ''), ('dns.id', ''), ('tcp.payload', '47:45:54:20:2f:63:6f:6d:70:6c:65:74:65:2f:73:65:61:72:63:68:3f:63:6c:69:65:6e:74:3d:63:68:72:6f:6d:65:26:68:6c:3d:65:6e:2d:55:53:26:71:3d:63:72:20:48:54:54:50:2f:31:2e:31:0d:0a:48:6f:73:74:3a:20:63:6c:69:65:6e:74:73:31:2e:67:6f:6f:67:6c:65:2e:63:61:0d:0a:43:6f:6e:6e:65:63:74:69:6f:6e:3a:20:6b:65:65:70:2d:61:6c:69:76:65:0d:0a:55:73:65:72:2d:41:67:65:6e:74:3a:20:4d:6f:7a:69:6c:6c:61:2f:35:2e:30:20:28:57:69:6e:64:6f:77:73:3b:20:55:3b:20:57:69:6e:64:6f:77:73:20:4e:54:20:36:2e:31:3b:20:65:6e:2d:55:53:29:20:41:70:70:6c:65:57:65:62:4b:69:74:2f:35:33:34:2e:31:30:20:28:4b:48:54:4d:4c:2c:20:6c:69:6b:65:20:47:65:63:6b:6f:29:20:43:68:72:6f:6d:65:2f:38:2e:30:2e:35:35:32:2e:32:33:37:20:53:61:66:61:72:69:2f:35:33:34:2e:31:30:0d:0a:41:63:63:65:70:74:2d:45:6e:63:6f:64:69:6e:67:3a:20:67:7a:69:70:2c:64:65:66:6c:61:74:65:2c:73:64:63:68:0d:0a:41:63:63:65:70:74:2d:4c:61:6e:67:75:61:67:65:3a:20:65:6e:2d:55:53:2c:65:6e:3b:71:3d:30:2e:38:0d:0a:41:63:63:65:70:74:2d:43:68:61:72:73:65:74:3a:20:49:53:4f:2d:38:38:35:39:2d:31:2c:75:74:66:2d:38:3b:71:3d:30:2e:37:2c:2a:3b:71:3d:30:2e:33:0d:0a:43:6f:6f:6b:69:65:3a:20:50:52:45:46:3d:49:44:3d:63:32:65:33:35:30:30:31:32:32:35:38:64:66:31:63:3a:55:3d:33:38:36:61:36:65:62:65:66:30:64:62:32:38:37:63:3a:46:46:3d:30:3a:54:4d:3d:31:32:39:34:31:36:34:32:39:34:3a:4c:4d:3d:31:32:39:34:31:36:34:32:39:34:3a:53:3d:62:63:75:77:4d:36:56:68:35:65:63:4b:78:71:6d:6b:3b:20:53:49:44:3d:44:51:41:41:41:4e:34:41:41:41:42:33:4d:77:37:68:53:41:58:6d:32:39:73:76:66:5a:51:78:52:68:61:45:56:4c:35:78:5f:37:4a:45:79:57:45:79:77:50:74:66:49:4b:6d:56:32:51:4d:43:5a:36:31:56:66:53:76:47:78:67:2d:57:43:77:53:37:4f:59:6e:45:6f:6e:61:76:64:52:65:69:54:67:5a:5f:33:4a:61:6c:63:50:79:49:6e:78:59:62:48:47:36:36:38:68:62:68:66:56:52:78:43:48:57:72:61:43:38:6c:4e:68:68:68:5a:76:43:34:35:4c:33:32:57:44:6a:6b:50:52:52:79:30:71:6d:6f:7a:5f:33:53:47:7a:44:44:67:75:6d:42:32:6d:67:79:6a:54:48:69:71:52:64:67:45:6d:6f:70:73:45:76:6f:75:6f:62:53:5a:44:52:78:69:78:58:64:41:4e:76:54:48:79:71:38:35:50:6d:56:6e:7a:4b:48:4b:5f:2d:78:37:68:56:64:59:68:75:34:34:4a:36:50:5f:6f:49:34:62:5a:57:6e:48:41:39:36:36:51:6e:61:37:33:71:35:59:4f:50:50:65:76:76:5a:51:56:58:38:46:37:31:6e:56:6a:44:6b:34:61:4a:4d:35:4b:68:6c:41:51:77:42:44:78:35:66:7a:72:56:39:57:6b:5f:52:5f:59:2d:65:67:7a:30:73:44:4c:39:6f:43:33:66:42:55:52:47:56:77:70:34:79:77:51:3b:20:48:53:49:44:3d:41:71:67:4d:33:4a:6c:7a:72:56:41:33:51:6b:69:79:7a:3b:20:4e:49:44:3d:34:33:3d:46:5f:6f:53:5a:57:79:6f:39:4e:69:61:64:6b:31:37:6d:36:35:51:74:4d:39:61:6c:42:4a:51:34:59:4c:30:42:30:79:41:50:37:31:72:75:4e:71:61:35:73:56:34:4a:4f:52:49:6d:73:51:6f:76:55:31:50:57:32:50:45:49:49:37:61:2d:35:4b:55:69:34:59:43:52:4d:43:65:79:74:75:68:69:77:6b:57:67:53:6c:57:74:48:45:41:6a:5f:6e:74:5f:45:46:38:79:38:34:4d:4e:6d:72:74:6d:52:7a:4b:39:4b:74:68:39:36:4e:34:2d:62:50:5a:0d:0a:0d:0a'), ('ssl.handshake.comp_methods_length', ''), ('tcp.reassembled.data', ''), ('udp.length', ''), ('browser.server_type.member', ''), ('smb.wct', ''), ('ssl.handshake.extensions_server_name_list_len', ''), ('ssl.record.version', ''), ('image-jfif.sos.se', ''), ('smb.flags.oplock', ''), ('smb.flags.receive_buffer', ''), ('dns.qry.type', ''), ('smb.nt.create_options.intermediate_buffering', ''), ('ssl.handshake.ciphersuite', ''), ('http.request.uri.path', '/complete/search'), ('smb.file_attribute.device', ''), ('icmp.resp_to', ''), ('rss.channel.image.url', ''), ('smb.access.read_ea', ''), ('browser.server_type.browser.potential', ''), ('tcp.window_size', '16288'), ('smb.access.write_ea', ''), ('smb.time', ''), ('tcp.analysis.window_full', ''), ('ssl.handshake.extension.data', ''), ('http.prev_response_in', ''), ('nbns.count.auth_rr', ''), ('image-jfif.sos.al', ''), ('dns.flags.response', ''), ('smb.po', ''), ('browser.server_type.server', ''), ('smb.file_attribute.directory', ''), ('png.chunk.flag.ancillary', ''), ('smb.flags.notify', ''), ('smb.file_attribute.temporary', ''), ('ssl.app_data', ''), ('dns.flags.tentative', ''), ('http.prev_request_in', ''), ('image-gif.image.left', ''), ('rss.channel.item', ''), ('png.chunk.len', ''), ('browser.os_major', ''), ('rss.channel.image', ''), ('png.srgb.intent', ''), ('xml.xmlpi.xml.encoding', ''), ('smb.share_access', ''), ('tcp.option_kind', ''), ('ssl.handshake.extensions_status_request_responder_ids_len', ''), ('tcp.analysis.push_bytes_sent', '943'), ('smb.file_attribute.compressed', ''), ('nbdgm.src.port', ''), ('xml.xmlpi.xml', ''), ('smb.file_attribute.not_content_indexed', ''), ('nbns.flags.opcode', ''), ('_score', ''), ('tcp.options.sack_le', ''), ('tcp.options.mss', ''), ('nbns.flags.recdesired', ''), ('arp.opcode', ''), ('udp.srcport', ''), ('bootp.hw.mac_addr', ''), ('nbdgm.type', ''), ('tcp.flags.str', 'Â·Â·Â·Â·Â·Â·Â·APÂ·Â·Â·'), ('smb.fid', ''), ('http.chat', ''), ('udp.checksum', ''), ('image-jfif.sos.ss', ''), ('eth.addr_resolved', '40:61:86:9a:f1:f5'), ('smb.rfid', ''), ('tcp.analysis.duplicate_ack', ''), ('image-gif.local.color_map.ordered', ''), ('image-jfif.entropy_coded_segment', ''), ('rss.channel.item.link', ''), ('tcp.hdr_len', '20'), ('snmp.value.null', ''), ('image-jfif.length', ''), ('tcp.analysis.initial_rtt', ''), ('ssl.handshake.extensions_npn', ''), ('ssl.handshake.challenge', ''), ('smb.nt.create_options.create_tree_connection', ''), ('smb.security.flags', ''), ('ip.dst', '72.14.213.138'), ('tcp.options.sack_perm', ''), ('ip.flags.rb', '0'), ('media.type', ''), ('smb.echo.count', ''), ('http.cookie_pair', 'NID=43=F_oSZWyo9Niadk17m65QtM9alBJQ4YL0B0yAP71ruNqa5sV4JORImsQovU1PW2PEII7a-5KUi4YCRMCeytuhiwkWgSlWtHEAj_nt_EF8y84MNmrtmRzK9Kth96N4-bPZ'), ('smb.nt.create_options.no_ea_knowledge', ''), ('icmp.code', ''), ('tcp.analysis.ack_lost_segment', ''), ('tcp.analysis.lost_segment', ''), ('browser.server_type.browser.backup', ''), ('smb.access.write_owner', ''), ('http.accept', ''), ('more testing', ''), ('tcp.analysis.acks_frame', ''), ('udp.stream', ''), ('eth.type', '0x00000800'), ('smb.tpc', ''), ('http.time', ''), ('smb.nt.create_options.random_access', ''), ('browser.period', ''), ('browser.server_type.xenix', ''), ('_index', 'packets-2020-04-07'), ('ssl.handshake.random_time', ''), ('dns.flags.recavail', ''), ('smb.nt.create_options.open_no_recall', ''), ('smb.alloc_size64', ''), ('image-gif.global.color_bpp', ''), ('smb.file_attribute.system', ''), ('ssl.handshake.challenge_length', ''), ('bootp.cookie', ''), ('smb.flags2.esn', ''), ('bootp.server', ''), ('smb.security.flags.effective_only', ''), ('png.chunk.crc', ''), ('tcp.flags.fin', '0'), ('nbns.class', ''), ('smb.error_code', ''), ('smb.nt.create_options.open_for_free_space_query', ''), ('smb.fid.closed_in', ''), ('eth.dst_resolved', '00:1a:8c:15:f9:80'), ('tcp.flags.push', '1'), ('_type', 'pcap_file'), ('http.request.method', 'GET'), ('smb.signature', ''), ('http.connection', 'keep-alive'), ('image-gif.extension.label', ''), ('image-jfif.sof.c_i', ''), ('png.ihdr.colour_type', ''), ('json.value.number', ''), ('tcp.options.nop', ''), ('nbns.type', ''), ('smb.flags2.ea', ''), ('arp.dst.hw_mac', ''), ('rtcp.psfb.fmt', ''), ('nbdgm.destination_name', ''), ('http.server', ''), ('smb.response_to', ''), ('dns.count.answers', ''), ('smb.access_mask', ''), ('smb.flags.caseless', ''), ('nbns.flags.truncated', ''), ('dns.response_to', ''), ('icmp.seq', ''), ('tcp.len', '943'), ('image-gif.global.color_map', ''), ('tcp.analysis.out_of_order', ''), ('http.request.line', 'Cookie: PREF=ID=c2e350012258df1c:U=386a6ebef0db287c:FF=0:TM=1294164294:LM=1294164294:S=bcuwM6Vh5ecKxqmk; SID=DQAAAN4AAAB3Mw7hSAXm29svfZQxRhaEVL5x_7JEyWEywPtfIKmV2QMCZ61VfSvGxg-WCwS7OYnEonavdReiTgZ_3JalcPyInxYbHG668hbhfVRxCHWraC8lNhhhZvC45L32WDjkPRRy0qmoz_3SGzDDgumB2mgyjTHiqRdgEmopsEvouobSZDRxixXdANvTHyq85PmVnzKHK_-x7hVdYhu44J6P_oI4bZWnHA966Qna73q5YOPPevvZQVX8F71nVjDk4aJM5KhlAQwBDx5fzrV9Wk_R_Y-egz0sDL9oC3fBURGVwp4ywQ; HSID=AqgM3JlzrVA3Qkiyz; NID=43=F_oSZWyo9Niadk17m65QtM9alBJQ4YL0B0yAP71ruNqa5sV4JORImsQovU1PW2PEII7a-5KUi4YCRMCeytuhiwkWgSlWtHEAj_nt_EF8y84MNmrtmRzK9Kth96N4-bPZ\\r\\n'), ('http.referer', ''), ('smb.last_write.time', ''), ('rss.channel.item.enclosure.type', ''), ('http.request.uri', '/complete/search?client=chrome&hl=en-US&q=cr'), ('rss.channel.copyright', ''), ('bootp.ip.server', ''), ('mailslot.priority', ''), ('http.last_modified', ''), ('I find the term packet sniffer dirty.', ''), ('bootp.dhcp', ''), ('dns.count.labels', ''), ('smb.flags2.dfs', ''), ('bootp.option.end', ''), ('tcp.segment_data', ''), ('rtcp.version', ''), ('urlencoded-form.key', ''), ('udp.possible_traceroute', ''), ('nbdgm.pkt_offset', ''), ('ip.host', '72.14.213.138'), ('bootp.ip.client', ''), ('png.chunk.flag.private', ''), ('dns.time', ''), ('smb.create_flags', ''), ('frame.len', '997'), ('rss.channel.item.title', ''), ('frame.protocols', 'eth:ethertype:ip:tcp:http'), ('arp.src.hw_mac', ''), ('dns.flags.authenticated', ''), ('dns.cname', ''), ('tcp.options.mss_val', ''), ('png.ihdr.filter_method', ''), ('icmp.checksum.status', ''), ('tcp.flags.ack', '1'), ('http.host', 'clients1.google.ca'), ('_ws.expert.severity', '2097152'), ('bootp.file', ''), ('bootp.secs', ''), ('snmp.name', ''), ('dns.flags.checkdisable', ''), ('nbdgm.dgram_len', ''), ('smb.error_class', ''), ('smb.tdc', ''), ('icmp.type', ''), ('http.response.line', ''), ('rtcp.length', ''), ('ssl.record.length', ''), ('tcp.time_delta', '0.000000000'), ('icmp.checksum', ''), ('dns.resp.ttl', ''), ('ip.ttl', '128'), ('ssl.handshake.length', ''), ('png.ihdr.width', ''), ('smb.share.access.write', ''), ('http.subdissector_failed', ''), ('png.chunk.flag.stc', ''), ('png.ihdr.interlace_method', ''), ('nat-pmp.external_port', ''), ('smb.data_offset', ''), ('smb.access.write_dac', ''), ('image-gif.global.color_map.ordered', ''), ('rss.channel.item.guid.ispermalink', ''), ('ip.checksum', '0x00009e7c'), ('smb.bcc', ''), ('xml.unknown', ''), ('frame.time_epoch', '1295981542.484409000'), ('http.date', ''), ('png.chunk.type', ''), ('smb.mdc', ''), ('smb.access.delete', ''), ('http.accept_encoding', 'gzip,deflate,sdch'), ('smb.dc', ''), ('data.len', ''), ('tcp.flags.urg', '0'), ('smb.transaction.flags.dtid', ''), ('rtcp.padding', ''), ('browser.server_type.ntw', ''), ('smb.nt.create_options.backup_intent', ''), ('tcp.options.sack', ''), ('tcp.options', ''), ('smb.file_attribute.offline', ''), ('smb.create.disposition', ''), ('smb.flags2', ''), ('tcp.options.wscale.multiplier', ''), ('tcp.analysis.rto', ''), ('browser.server_type.print', ''), ('browser.server_type.nts', ''), ('image-gif.image.code_size', ''), ('ssl.record.content_type', ''), ('smb.file_attribute.normal', ''), ('rss.channel.item.guid', ''), ('smb.fid.opened_in', ''), ('http.request.full_uri', 'http://clients1.google.ca/complete/search?client=chrome&hl=en-US&q=cr'), ('rss.channel.item.pubdate', ''), ('rss.channel.item.category', ''), ('bootp.hw.len', ''), ('smb.nt_status', ''), ('dns.flags.truncated', ''), ('ssl.alert_message', ''), ('nbns.id', ''), ('browser.server_type.novell', ''), ('http.response.phrase', ''), ('image-jfif.sof.v_i', ''), ('udp.dstport', ''), ('tcp.options.wscale', ''), ('nbss.type', ''), ('nbdgm.dgram_id', ''), ('image-jfif.sof.h_i', ''), ('smb.file_attribute', ''), ('smb.file_attribute.sparse', ''), ('bootp.flags.reserved', ''), ('smb.share.access.read', ''), ('png.phys.unit', ''), ('http.request.version', 'HTTP/1.1'), ('dns.ns', ''), ('tcp.options.sack.count', ''), ('eth.addr', '40:61:86:9a:f1:f5'), ('tcp.options.wscale.shift', ''), ('snmp.request_id', '')])\n"
     ]
    }
   ],
   "source": [
    "# So simply use an existing python file and class already in the `funcs` directory\n",
    "# or create a new one, and make sure the class subclasses `Features`\n",
    "# and the function signatures take in rows and return rows\n",
    "# the above example as a helper function `get_columns` that lets you provide a list of fields and the rows\n",
    "# and returns the rows with only those fields\n",
    "\n",
    "# You don't need to use the helper function, but the returns rows should be a list of dictionaries\n",
    "# just like the input of rows is (the same thing you get back from a CSV DictReader)\n",
    "\n",
    "# Here's a sample to test:\n",
    "from networkml.featurizers.csv_to_features import CSVToFeatures\n",
    "instance = CSVToFeatures()\n",
    "rows = list(csv.DictReader(instance.get_reader('trace_ab12_2001-01-01_02_03-client-ip-1-2-3-4.pcap.csv.gz', True)))\n",
    "print(rows[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eth.src': '40:61:86:9a:f1:f5', 'eth.dst': '00:1a:8c:15:f9:80'}\n"
     ]
    }
   ],
   "source": [
    "from networkml.featurizers.features import Features\n",
    "\n",
    "class Flow(Features):\n",
    "\n",
    "    def example_simple(self, rows):\n",
    "        fields = ['eth.src', 'eth.dst']\n",
    "        return self.get_columns(fields, rows)\n",
    "\n",
    "flow = Flow()\n",
    "new_rows = flow.example_simple(rows)\n",
    "print(new_rows[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "odict_keys(['ip.id', 'dns.qry.name.len', 'smb.access.delete_child', 'smb.access.read_control', 'dns.resp.len', 'smb.nt.create_options.non_directory', 'smb.pc', 'bootp.option.type', 'ssl.handshake.extensions_status_request_exts_len', 'tcp.window_size_scalefactor', 'tcp.connection.fin', 'nbdgm.src.ip', 'ip.flags', 'smb.nt.create_options.reserve_opfilter', 'eth.lg', 'http.request.uri.query.parameter', 'http.chunk_size', 'rss.channel.image.title', 'smb.pid.high', 'nbns.count.answers', 'ssl.handshake.extensions_status_request_type', 'frame.time_relative', 'tcp.checksum', 'ssl.handshake.version', 'snmp.error_index', 'ssl.handshake.extensions_ec_point_formats_length', 'png.phys.vert', 'browser.server_type.dialin', 'dns.retransmit_request', 'frame.cap_len', 'smb.timeout', 'arp.src.proto_ipv4', 'rss.version', 'smb.nt.create_options.sync_io_nonalert', 'smb.nt.create_options.no_compression', 'arp.hw.type', 'nat-pmp.opcode', 'ssl.handshake.extensions_reneg_info_len', 'smb.security.flags.context_tracking', 'ssl.handshake.random', 'ip.src', 'ssl.handshake.random_bytes', 'http.set_cookie', 'xml.tag', 'browser.proto_minor', 'rss.channel.link', 'ip.flags.mf', 'image-jfif.sos.ac_entropy_selector', 'dns.resp.type', 'ip.dsfield.dscp', 'ssl.handshake.cipher_suites_length', 'nbns.count.queries', 'ssl.handshake.type', 'tcp.option_len', 'tcp.analysis.rto_frame', 'dns.count.auth_rr', 'http.file_data', 'http.transfer_encoding', 'smb.access.system_security', 'tcp.time_relative', 'tcp.port', 'http.request_in', 'browser.command', 'tcp.connection.rst', 'arp.dst.proto_ipv4', 'udp.checksum.status', 'ssl.handshake', 'icmp.seq_le', 'ssl.handshake.extensions_server_name', 'smb.andxoffset', 'rss.channel.item.enclosure.length', 'xml.xmlpi.xml.version', 'ip.checksum.status', 'rss.channel.language', 'tcp.srcport', 'image-gif.image.height', 'smb.flags.lock', 'smb.nt.create_options.open_by_fileid', 'bootp.flags', 'http.response.code.desc', 'smb.echo.data', 'ip.dst_host', 'tcp.window_size_value', 'image-gif.global.color_map.present', 'smb.cmd', 'smb.nt.create_options.write_through', '_ws.malformed', 'rss.channel.item.description', 'smb.flags2.nt_error', 'nbns.count.add_rr', 'tcp.stream', 'image-jfif.sos.component_selector', 'browser.server_type', 'ssl.handshake.extension.len', '_ws.expert.message', 'data.data', 'smb.mid', 'smb.flags2.compressed', 'smb.trans_name', 'nbdgm.source_name', 'tcp.flags.ns', 'arp.proto.type', 'dns.flags.authoritative', 'eth.src_resolved', 'http.response_number', 'tcp.analysis.ack_rtt', 'ip.ttl.too_small', 'tcp.dstport', 'mailslot.opcode', 'data-text-lines', 'bootp.type', 'dns.count.add_rr', 'tcp.flags.cwr', 'tcp.analysis.bytes_in_flight', 'ip.dsfield.ecn', 'smb.share.access.delete', 'nbns.name', 'browser.server_type.apple', 'smb.file_attribute.reparse', 'ssl.handshake.extension.type', '_ws.malformed.expert', 'bootp.option.padding', 'smb.file', 'ssl.handshake.session_id_length', 'ip.proto', 'tcp.analysis.duplicate_ack_frame', 'http.response_for.uri', 'rss.channel', 'smb.file_attribute.encrypted', 'ssl.handshake.cipher_spec_len', 'browser.server_type.domain_controller', 'http.request_number', 'browser.server', 'png.chunk.data', 'dns.flags.opcode', 'icmp.unused', 'image-gif.image.top', 'arp.proto.size', 'image-gif.image.width', 'bootp.ip.your', 'ssl.handshake.cipherspec', 'image-gif.image_background_index', 'browser.server_type.wfw', 'frame.time_delta_displayed', 'ssl.handshake.extensions_supported_group', 'smb.flags2.reparse_path', 'bootp.hops', 'smb.msc', 'smb.flags2.long_names_used', 'image-jfif.remain_seg_data', 'smb.access.maximum_allowed', 'dns.flags', 'icmp.ident', 'tcp.options.eol', 'nbss.flags.e', 'bootp.flags.bc', 'image-jfif.sof.precision', 'png.ihdr.compression_method', 'snmp.error_status', 'rss.channel.description', 'ip.hdr_len', 'ip.len', 'snmp.variable_bindings', 'browser.server_type.osf', 'image-gif.screen.height', 'bootp.hw.type', 'smb.server_component', 'frame.encap_type', 'tcp.reassembled.length', 'frame.number', 'bootp.hw.addr_padding', 'smb.create_options', 'nat-pmp.reserved', 'nbns.flags.broadcast', 'tcp.seq', 'nbdgm.flags', 'image-gif.global.bpp', 'smb.access.synchronize', 'tcp.analysis.window_update', 'arp.hw.size', 'smb.transaction.flags.owt', 'tcp.flags', 'browser.server_type.w95', 'nat-pmp.version', 'tcp.flags.ecn', 'json.key', 'png.ihdr.bitdepth', 'http.request', 'http.content_encoding', 'mailslot.name', 'smb.sc', 'ssl.ignored_unknown_record', 'tcp.segment', 'browser.server_type.browser.master', 'frame.ignored', 'http.cache_control', 'dns.resp.class', 'rss.channel.lastbuilddate', 'image-jfif.sof.nf', 'image-gif.version', 'browser.sig', 'bootp.id', 'smb.flags2.long_names_allowed', 'frame.offset_shift', 'image-gif.local.color_bpp', 'tcp.flags.syn', 'udp.port', 'ssl.handshake.extensions_server_name_type', 'icmp.resptime', 'smb.access.generic_all', 'smb.nt.create_options.eight_dot_three_only', 'smb.nt.create.oplock', 'dns.flags.recdesired', 'http.response.version', 'mailslot.size', 'tcp.ack.nonzero', 'rtcp.pt', 'dns.qry.name', 'nat-pmp.internal_port', 'nbdgm.next', 'browser.server_type.workstation', 'png.gama.gamma', 'image-jfif.sos.dc_entropy_selector', 'mailslot.class', 'nbdgm.node_type', 'browser.proto_major', 'image-gif.data_block', 'tcp.nxtseq', 'http.user_agent', 'smb.access.write_attributes', 'dns.retransmit_request_in', 'rss.channel.item.enclosure.url', 'image-jfif.marker', 'nbss.length', 'smb.impersonation.level', 'bootp.ip.relay', 'ip.addr', 'smb.access.generic_write', 'smb.flags2.roe', 'ssl.handshake.extensions_server_name_len', 'smb.access.read_attributes', 'smb.nt.create_options.sequential_only', 'browser.server_type.local', 'tcp.segment.count', 'smb.flags2.sec_sig', 'browser.os_minor', 'smb.nt.create_options.sync_io_alert', 'tcp.connection.sack', 'eth.ig', 'rss.channel.title', 'dns.flags.rcode', 'browser.server_type.browser.domain_master', 'smb.flags2.sec_sig_required', 'snmp.data', 'image-jfif.sof.lines', 'http.content_length', 'image-jfif.sos.ns', 'nbns.flags', 'snmp.community', 'browser.server_type.dfs', 'dns.flags.conflict', 'image-jfif.sos.ah', 'ssl.handshake.extensions_npn_str_len', 'smb.file_name_len', 'smb.tid', 'nbns.flags.response', 'dns.qry.class', 'browser.server_type.backup_controller', 'ssl.handshake.session_id', 'png.ihdr.height', 'image-gif.screen.width', 'snmp.value.octets', 'rss.channel.image.link', 'rss.channel.ttl', 'tcp.checksum.status', 'browser.comment', 'smb.echo.seq_num', 'ssl.handshake.comp_method', 'image-gif.end', 'image-jfif.sof.samples_per_line', 'dns.flags.z', 'eth.src', 'browser.update_count', 'http.response', 'smb.access.write', 'xml.attribute', 'rtcp.mediassrc', 'tcp.flags.reset', 'dns.resp.name', 'smb.flags', 'frame.time', 'urlencoded-form.value', 'browser.server_type.domainenum', 'snmp.value.int', 'frame.time_delta', 'smb.nt.create_options.open_reparse_point', 'image-jfif.sof.tq_i', 'smb.file_attribute.hidden', 'ssl', 'http.content_type', 'ssl.handshake.extensions_length', 'dns.a', 'dns.retransmission', '_ws.expert.group', 'smb.flags.canon', 'browser.windows_version', 'smb.transaction.flags', 'tcp.connection.syn', 'http.accept_language', 'tcp.analysis.retransmission', 'smb.mpc', 'tcp.urgent_pointer', 'http.request.uri.query', 'smb.nt.create_options.complete_if_oplocked', 'http.response.code', 'browser.server_type.vms', 'smb.nt.create_options.directory', 'smb.flags2.string', 'png.signature', 'tcp.analysis.duplicate_ack_num', 'smb.access.append', 'ip.dsfield', 'http.content_length_header', 'ssl.handshake.extensions_supported_groups_length', 'rtcp.senderssrc', 'rss.channel.item.enclosure', 'smb.file_attribute.read_only', 'smb.access.generic_read', 'xml.cdata', 'ip.frag_offset', 'smb.reserved', 'nbdgm.first', 'smb.access.generic_execute', 'smb.nt.create.batch_oplock', 'smb.file_attribute.volume', 'http.location', 'smb.nt.create.dir', 'image-gif.local.color_map.present', 'smb.nt.create_options.delete_on_close', 'smb.flags.response', 'ip.flags.df', 'smb.uid', 'ip.version', 'dns.count.queries', 'smb.access.execute', 'smb.nt.create.ext', 'eth.padding', 'tcp.ack', 'ip.src_host', 'nbss.flags', 'tcp.options.sack_re', 'nat-pmp.pml', 'tcp.flags.res', 'png.phys.horiz', 'snmp.version', 'browser.server_type.time', 'smb.access.read', 'http.cookie', 'frame.marked', 'eth.dst', 'ssl.handshake.extensions_ec_point_format', 'browser.server_type.sql', 'http.chunk_boundary', 'smb.file_attribute.archive', 'smb.pid', 'dns.id', 'tcp.payload', 'ssl.handshake.comp_methods_length', 'tcp.reassembled.data', 'udp.length', 'browser.server_type.member', 'smb.wct', 'ssl.handshake.extensions_server_name_list_len', 'ssl.record.version', 'image-jfif.sos.se', 'smb.flags.oplock', 'smb.flags.receive_buffer', 'dns.qry.type', 'smb.nt.create_options.intermediate_buffering', 'ssl.handshake.ciphersuite', 'http.request.uri.path', 'smb.file_attribute.device', 'icmp.resp_to', 'rss.channel.image.url', 'smb.access.read_ea', 'browser.server_type.browser.potential', 'tcp.window_size', 'smb.access.write_ea', 'smb.time', 'tcp.analysis.window_full', 'ssl.handshake.extension.data', 'http.prev_response_in', 'nbns.count.auth_rr', 'image-jfif.sos.al', 'dns.flags.response', 'smb.po', 'browser.server_type.server', 'smb.file_attribute.directory', 'png.chunk.flag.ancillary', 'smb.flags.notify', 'smb.file_attribute.temporary', 'ssl.app_data', 'dns.flags.tentative', 'http.prev_request_in', 'image-gif.image.left', 'rss.channel.item', 'png.chunk.len', 'browser.os_major', 'rss.channel.image', 'png.srgb.intent', 'xml.xmlpi.xml.encoding', 'smb.share_access', 'tcp.option_kind', 'ssl.handshake.extensions_status_request_responder_ids_len', 'tcp.analysis.push_bytes_sent', 'smb.file_attribute.compressed', 'nbdgm.src.port', 'xml.xmlpi.xml', 'smb.file_attribute.not_content_indexed', 'nbns.flags.opcode', '_score', 'tcp.options.sack_le', 'tcp.options.mss', 'nbns.flags.recdesired', 'arp.opcode', 'udp.srcport', 'bootp.hw.mac_addr', 'nbdgm.type', 'tcp.flags.str', 'smb.fid', 'http.chat', 'udp.checksum', 'image-jfif.sos.ss', 'eth.addr_resolved', 'smb.rfid', 'tcp.analysis.duplicate_ack', 'image-gif.local.color_map.ordered', 'image-jfif.entropy_coded_segment', 'rss.channel.item.link', 'tcp.hdr_len', 'snmp.value.null', 'image-jfif.length', 'tcp.analysis.initial_rtt', 'ssl.handshake.extensions_npn', 'ssl.handshake.challenge', 'smb.nt.create_options.create_tree_connection', 'smb.security.flags', 'ip.dst', 'tcp.options.sack_perm', 'ip.flags.rb', 'media.type', 'smb.echo.count', 'http.cookie_pair', 'smb.nt.create_options.no_ea_knowledge', 'icmp.code', 'tcp.analysis.ack_lost_segment', 'tcp.analysis.lost_segment', 'browser.server_type.browser.backup', 'smb.access.write_owner', 'http.accept', 'more testing', 'tcp.analysis.acks_frame', 'udp.stream', 'eth.type', 'smb.tpc', 'http.time', 'smb.nt.create_options.random_access', 'browser.period', 'browser.server_type.xenix', '_index', 'ssl.handshake.random_time', 'dns.flags.recavail', 'smb.nt.create_options.open_no_recall', 'smb.alloc_size64', 'image-gif.global.color_bpp', 'smb.file_attribute.system', 'ssl.handshake.challenge_length', 'bootp.cookie', 'smb.flags2.esn', 'bootp.server', 'smb.security.flags.effective_only', 'png.chunk.crc', 'tcp.flags.fin', 'nbns.class', 'smb.error_code', 'smb.nt.create_options.open_for_free_space_query', 'smb.fid.closed_in', 'eth.dst_resolved', 'tcp.flags.push', '_type', 'http.request.method', 'smb.signature', 'http.connection', 'image-gif.extension.label', 'image-jfif.sof.c_i', 'png.ihdr.colour_type', 'json.value.number', 'tcp.options.nop', 'nbns.type', 'smb.flags2.ea', 'arp.dst.hw_mac', 'rtcp.psfb.fmt', 'nbdgm.destination_name', 'http.server', 'smb.response_to', 'dns.count.answers', 'smb.access_mask', 'smb.flags.caseless', 'nbns.flags.truncated', 'dns.response_to', 'icmp.seq', 'tcp.len', 'image-gif.global.color_map', 'tcp.analysis.out_of_order', 'http.request.line', 'http.referer', 'smb.last_write.time', 'rss.channel.item.enclosure.type', 'http.request.uri', 'rss.channel.copyright', 'bootp.ip.server', 'mailslot.priority', 'http.last_modified', 'I find the term packet sniffer dirty.', 'bootp.dhcp', 'dns.count.labels', 'smb.flags2.dfs', 'bootp.option.end', 'tcp.segment_data', 'rtcp.version', 'urlencoded-form.key', 'udp.possible_traceroute', 'nbdgm.pkt_offset', 'ip.host', 'bootp.ip.client', 'png.chunk.flag.private', 'dns.time', 'smb.create_flags', 'frame.len', 'rss.channel.item.title', 'frame.protocols', 'arp.src.hw_mac', 'dns.flags.authenticated', 'dns.cname', 'tcp.options.mss_val', 'png.ihdr.filter_method', 'icmp.checksum.status', 'tcp.flags.ack', 'http.host', '_ws.expert.severity', 'bootp.file', 'bootp.secs', 'snmp.name', 'dns.flags.checkdisable', 'nbdgm.dgram_len', 'smb.error_class', 'smb.tdc', 'icmp.type', 'http.response.line', 'rtcp.length', 'ssl.record.length', 'tcp.time_delta', 'icmp.checksum', 'dns.resp.ttl', 'ip.ttl', 'ssl.handshake.length', 'png.ihdr.width', 'smb.share.access.write', 'http.subdissector_failed', 'png.chunk.flag.stc', 'png.ihdr.interlace_method', 'nat-pmp.external_port', 'smb.data_offset', 'smb.access.write_dac', 'image-gif.global.color_map.ordered', 'rss.channel.item.guid.ispermalink', 'ip.checksum', 'smb.bcc', 'xml.unknown', 'frame.time_epoch', 'http.date', 'png.chunk.type', 'smb.mdc', 'smb.access.delete', 'http.accept_encoding', 'smb.dc', 'data.len', 'tcp.flags.urg', 'smb.transaction.flags.dtid', 'rtcp.padding', 'browser.server_type.ntw', 'smb.nt.create_options.backup_intent', 'tcp.options.sack', 'tcp.options', 'smb.file_attribute.offline', 'smb.create.disposition', 'smb.flags2', 'tcp.options.wscale.multiplier', 'tcp.analysis.rto', 'browser.server_type.print', 'browser.server_type.nts', 'image-gif.image.code_size', 'ssl.record.content_type', 'smb.file_attribute.normal', 'rss.channel.item.guid', 'smb.fid.opened_in', 'http.request.full_uri', 'rss.channel.item.pubdate', 'rss.channel.item.category', 'bootp.hw.len', 'smb.nt_status', 'dns.flags.truncated', 'ssl.alert_message', 'nbns.id', 'browser.server_type.novell', 'http.response.phrase', 'image-jfif.sof.v_i', 'udp.dstport', 'tcp.options.wscale', 'nbss.type', 'nbdgm.dgram_id', 'image-jfif.sof.h_i', 'smb.file_attribute', 'smb.file_attribute.sparse', 'bootp.flags.reserved', 'smb.share.access.read', 'png.phys.unit', 'http.request.version', 'dns.ns', 'tcp.options.sack.count', 'eth.addr', 'tcp.options.wscale.shift', 'snmp.request_id'])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# great, but how did we know the fields?\n",
    "rows[0].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of original records: 14261\n",
      "\n",
      "Fields/Values in first record (row): {'eth.src': '40:61:86:9a:f1:f5', 'eth.dst': '00:14:5e:6b:72:00', 'Combined IPs': '192.168.3.131:192.168.3.90'}\n",
      "\n",
      "Number of records after deduplication: 432\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# what if we want to create a new field/column or reduce the number of records/rows?\n",
    "from networkml.featurizers.features import Features\n",
    "\n",
    "class NewColumn(Features):\n",
    "\n",
    "    def example_modify(self, rows):\n",
    "        # reduce rows first as needed\n",
    "        fields = ['layers', 'eth.src', 'eth.dst', 'ip.src', 'ip.dst']\n",
    "        rows = self.get_columns(fields, rows)\n",
    "        \n",
    "        # create new columns using existing column info\n",
    "        last_layer = 'Last Layer'\n",
    "        combined_ips = 'Combined IPs'\n",
    "        # each row is a dict\n",
    "        for row in rows:\n",
    "            # not all rows are guaranteed to have 'layers'\n",
    "            if 'layers' in row:  \n",
    "                # get the last element in the stringified list and clean it up\n",
    "                row[last_layer] = row['layers'].split('<')[-1][:-2].split()[0]\n",
    "            \n",
    "            # not all rows are guaranteed to have 'ip.src' and ip.dst\n",
    "            if 'ip.src' in row and 'ip.dst' in row:\n",
    "                # combine two fields with a colon, making a new field\n",
    "                row[combined_ips] = row['ip.src']+':'+row['ip.dst']\n",
    "                # remove ip.src and ip.dst now that we have them\n",
    "                del row['ip.src']\n",
    "                del row['ip.dst']\n",
    "        \n",
    "        # remove duplicate rows\n",
    "        rows = [dict(t) for t in {tuple(d.items()) for d in rows}]\n",
    "        return rows\n",
    "\n",
    "print(f'Number of original records: {len(rows)}\\n')\n",
    "ncol = NewColumn()\n",
    "new_rows = ncol.example_modify(rows)\n",
    "print(f'Fields/Values in first record (row): {new_rows[0]}\\n')\n",
    "print(f'Number of records after deduplication: {len(new_rows)}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
